Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
b6f4d04
recreate commits from #11 (Northern Sámi TN)
jimregan Sep 3, 2023
3e61af8
Northern Sami symbol names
kathiasi Mar 28, 2023
03eb0c0
Add Northern Sami (se) text normalization
jimregan Aug 21, 2026
c3706f4
remove Swedishness
jimregan Sep 7, 2026
e32e5ba
Merge branch 'main' into northern-sami-tn3
jimregan Sep 19, 2026
8eeae49
update year
jimregan Oct 13, 2026
526afdd
restrict separators if not deterministic; also remove a stray comment
jimregan Oct 13, 2026
04a61f2
replace single-entry file with string_map
jimregan Oct 13, 2026
e879780
update year
jimregan Oct 13, 2026
c95682a
change arg name
jimregan Oct 13, 2026
e2a77f0
move asserts outside loop
jimregan Oct 13, 2026
2bddcaa
make acceptors
jimregan Oct 13, 2026
16b02dc
remove f-strings, use .copy()
jimregan Oct 13, 2026
29e9e4f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 13, 2026
b1a96b3
NEMO_SPACE (+ delete space, explicit transduction of b. -> beaivvis
jimregan Oct 13, 2026
fa38155
Merge branch 'northern-sami-tn3' of github.com:jimregan/NeMo-text-pro…
jimregan Oct 13, 2026
ce12a5d
insert_space
jimregan Oct 13, 2026
0dd5451
move to tsv
jimregan Oct 13, 2026
8c92d05
move from utils to graph_utils
jimregan Oct 13, 2026
3828759
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 13, 2026
6f02a9f
never got around to using this anyway
jimregan Oct 13, 2026
8de6879
use unicodedata instead of a table
jimregan Oct 14, 2026
ef8eb96
that did not need to be non-deterministic
jimregan Oct 14, 2026
ac9fe18
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 14, 2026
dec8256
make explicit copies of teens and ties, remove aliases
jimregan Oct 14, 2026
d587597
Merge branch 'northern-sami-tn3' of github.com:jimregan/NeMo-text-pro…
jimregan Oct 14, 2026
bb7b7f3
generate greek mappings from unicode data; valid according to https:/…
jimregan Oct 14, 2026
bd6ab3b
update year
jimregan Oct 16, 2026
7794689
remove from Jenkinsfile
jimregan Oct 19, 2026
bd1a95b
sync with current main
jimregan Oct 19, 2026
7956a18
nothing in these files that I could have had copyright in
jimregan Oct 19, 2026
afca0a2
nothing in these files that I could have had copyright in
jimregan Oct 19, 2026
89fbbe3
but this one was me
jimregan Oct 19, 2026
0486ec1
optimize
jimregan Oct 19, 2026
06a03b8
switch from tsv to string_file, change the way the dict is created
jimregan Oct 19, 2026
d4e6a94
delete duplicate files; illative and locative are same as genitive in…
jimregan Oct 19, 2026
e79b3a0
Ok. These are *attribution* lines, not copyright notices, so let's no…
jimregan Oct 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion nemo_text_processing/text_normalization/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ def __init__(
elif lang == 'fr':
from nemo_text_processing.text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.fr.verbalizers.verbalize_final import VerbalizeFinalFst
elif lang == 'se':
from nemo_text_processing.text_normalization.se.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.se.verbalizers.verbalize_final import VerbalizeFinalFst
elif lang == 'sv':
from nemo_text_processing.text_normalization.sv.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.sv.verbalizers.verbalize_final import VerbalizeFinalFst
Expand Down Expand Up @@ -745,7 +748,7 @@ def parse_args():
parser.add_argument(
"--language",
help="language",
choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"],
choices=["en", "de", "es", "fr", "hu", "se", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"],
default="en",
type=str,
)
Expand Down
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/se/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/se/data/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/se/data/dates/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
o.Kr. ovdal Kristtusa
m.Kr. maŋŋel Kristtusa
oKr. ovdal Kristtusa
mKr. maŋŋel Kristtusa
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
ođđajagem ođđajagemánnu
ođđajagim ođđajagimánnu
guovvam guovvamánnu
njukčam njukčamánnu
cuoŋom cuoŋománnu
miessem miessemánnu
geassem geassemánnu
suoidnem suoidnemánnu
borgem borgemánnu
čakcam čakčamánnu
golggotm golggotmánnu
skábmam skábmamánnu
juovlam juovlamánnu
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
ođđajagemánu ođđajagemánnu
guovvamánu guovvamánnu
njukčamánu njukčamánnu
cuoŋománu cuoŋománnu
miessemánu miessemánnu
geassemánu geassemánnu
suoidnemánu suoidnemánnu
borgemánu borgemánnu
čakčamánu čakčamánnu
golggotmánu golggotmánnu
skábmamánu skábmamánnu
juovlamánu juovlamánnu
12 changes: 12 additions & 0 deletions nemo_text_processing/text_normalization/se/data/dates/numbers.tsv
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
1 ođđajagimánnu
2 guovvamánnu
3 njukčamánnu
4 cuoŋománnu
5 miessemánnu
6 geassemánnu
7 suoidnemánnu
8 borgemánnu
9 čakčamánnu
10 golggotmánnu
11 skábmamánnu
12 juovlamánnu
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
.com čuokkis com
.uk čuokkis u k
.fr čuokkis f r
.net čuokkis net
.br čuokkis b r
.in čuokkis i n
.ru čuokkis r u
.de čuokkis d e
.it čuokkis i t
.se čuokkis s e
.no čuokkis n o
.fi čuokkis f i
.es čuokkis e s
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
gmail g mail
nvidia
outlook
hotmail
yahoo
live
yandex
orange
wanadoo
web
comcast
aol
sametinget
samediggi
sámediggi
ijahisidja ijahis idja
sajos
salliparikka salli parikka
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
. čuokkis
: duppalčuokkis
- sáhcu
! čuorvvasmearka
; beallečuokkis
' apostrofa
/ finjusáhcu
? gažaldatmearka
, rihkku
_ deattuhusmearka
# fáddágilkor
$ dollár
% proseanta
& ja
* asterisk
+ plussa
= ovttamađodatmearka
^ sirkumfleaksa
{ gurut bárroruohtu
} olgeš bárroruohtu
~ tilde
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
nom_pl t
gen_sg a
gen_pl id
ill_sg i
ill_pl ide
loc_sg s
loc_pl in
com_sg in
com_pl iguin
ess n
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
ess jahkin
com_pl jagiiguin
com_sg jagiin
acc_sg jagi
gen_sg jage
gen_pl jagiid
ill_pl jagiide
ill_sg jahkái
loc_sg jagis
nom_pl jagit
nom_sg jagi
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
nom_pl dat
gen_sg da
acc_sg da
gen_pl diid
acc_pl diid
ill_sg dii
ill_pl diidda
loc_sg dis
loc_pl diin
com_sg diin
com_pl diiguin
ess din
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
:t at
:a a
:id iid
:i ii
:ida iidda
:s is
:in iin
:iguin iiguin
:n in
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
ess vahkkun
com_pl vahkuiguin
com_sg vahkuin
gen_sg vahku
gen_pl vahkuid
ill_pl vahkuide
ill_sg vahkkui
loc_sg vahkus
nom_pl vahkut
nom_sg vahkku
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
+ plussa
- minus
/ juohkit
÷ juohkit
: juohkit
× geardde
* geardde
· geardde
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
+ plussa
- sáhcu
* asterisk
/ finjusáhcu
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
+ plussa
- minus
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
m mehtera
km kilomehtera
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
km/d kilomehtera diimmus
km/dii. kilomehtera diimmus
km/h kilomehtera diimmus
m/s mehtera sekunddas
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
cm² kvadráhttasentimehter
cm2 kvadráhttasentimehter
cm sentimehter
mm² kvadráhttamillimehter
mm2 kvadráhttamillimehter
mm millimehter
cm³ guđahassentimehter
cm3 guđahassentimehter
mm³ guđahasmillimehter
mm3 guđahasmillimehter
kg kilográmma
˚C celsiusgráda
°C celsiusgráda
°F Fahrenheit-gráda
A ampera
cal. kaloriija
cg centigrámma
cl centilihtar
dB desibel
dg desigrámma
dl desilihtar
dm desimehter
g. grámma
g grámma
hg hektográmma
hl hektolihtar
Hz hertz
J joule
K Kelvin-gráda
kB kilobyte
km kilomehter
kV kilovolta
kVA kilovoltampere
kW kilowatt
kWh kilowattdiibmu
l lihtar
m mehter
m² njealjehasmehter
MB megabyte
mg milligrámma
MHz megahertz
ml millilihtar
mV millivolt
MW megawatt
mW milliwatt
N newton
t. tonna
t tonna
V volt
W watt
yd yard
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/se/data/money/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
sek ruoŧa ruvdno
nok norgga ruvdno
kr ruvdno
bfr Belgia franc
Dkr Dánmárku ruvdno
DKK Dánmárku ruvdno
FIM Suoma márki
fr Fránkriikka franc
mk márki
Nkr norgga ruvdno
NOK norgga ruvdno
pta pesetas
rbl rubel
SEK ruoŧa ruvdno
sfr Šveicca franc
£ punda
e euro
eur euro
€ euro
₽ ruppal
$ dollár
USD dollár
GBP punda
E euro
MK márki
KR ruvdno
RU ruvdno
¥ yen
dollár
punda
pund punda
euro
yen
márki
czk čeahka ruvdno
kč čeahka ruvdno
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
$:a dollára
$:t dollárat
$:i dollárii
$:id dolláriid
$:ida dolláriidda
$:iguin dolláriiguin
$:in dolláriin
$:n dollárin
$:s dolláris
Loading