Upload all models and assets for da (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-germanic_north
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,14 +33,14 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.256
+    value: 4.557
   - name: best_isotropy
     type: isotropy
-    value: 0.7852
+    value: 0.7924
   - name: vocabulary_size
     type: vocab
-    value: 974563
-generated: 2025-12-29
+    value: 0
+generated: 2026-01-08
 ---
 
 # Danish - Wikilangs Models
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,73 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.425x | 3.39 | 0.1175% | 1,887,939 |
-| **16k** | 3.744x | 3.70 | 0.1284% | 1,727,173 |
-| **32k** | 4.025x | 3.98 | 0.1381% | 1,606,621 |
-| **64k** | 4.256x 🏆 | 4.21 | 0.1460% | 1,519,331 |
+| **8k** | 3.590x | 3.59 | 0.1227% | 1,644,330 |
+| **16k** | 3.953x | 3.95 | 0.1351% | 1,493,346 |
+| **32k** | 4.286x | 4.29 | 0.1465% | 1,377,449 |
+| **64k** | 4.557x 🏆 | 4.56 | 0.1558% | 1,295,305 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Se også 564 (tal)
-
- Begivenheder 
-
- Født 
-
- Dødsfald 
-
- Eksterne henvisninger 
-
-...`
+**Sample 1:** `Ole Bornemann henviser til: Oluf Bornemann – dansk-norsk biskop Ole Bornemann (r...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁se ▁også ▁ 5 6 4 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 16k | `▁se ▁også ▁ 5 6 4 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 32k | `▁se ▁også ▁ 5 6 4 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 64k | `▁se ▁også ▁ 5 6 4 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-
-**Sample 2:** `Rikuya Izutsu (født 10. februar 1994) er en japansk fodboldspiller.
+| 8k | `▁ole ▁bor nem ann ▁henviser ▁til : ▁oluf ▁bor nem ... (+27 more)` | 37 |
+| 16k | `▁ole ▁bor nemann ▁henviser ▁til : ▁oluf ▁bor nemann ▁– ... (+21 more)` | 31 |
+| 32k | `▁ole ▁bor nemann ▁henviser ▁til : ▁oluf ▁bor nemann ▁– ... (+20 more)` | 30 |
+| 64k | `▁ole ▁bornemann ▁henviser ▁til : ▁oluf ▁bornemann ▁– ▁dansk - ... (+16 more)` | 26 |
 
- Referencer...`
+**Sample 2:** `18. April er en dansk dokumentarfilm fra instrueret af Poul Meyer. Eksterne henv...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ri ku ya ▁i z ut su ▁( født ▁ ... (+23 more)` | 33 |
-| 16k | `▁ri ku ya ▁i z ut su ▁( født ▁ ... (+23 more)` | 33 |
-| 32k | `▁ri ku ya ▁iz ut su ▁( født ▁ 1 ... (+22 more)` | 32 |
-| 64k | `▁ri ku ya ▁iz utsu ▁( født ▁ 1 0 ... (+21 more)` | 31 |
-
-**Sample 3:** `Se også 598 (tal)
-
- Begivenheder 
-
- Født 
-
- Dødsfald 
-
- Eksterne henvisninger 
+| 8k | `▁ 1 8 . ▁april ▁er ▁en ▁dansk ▁dokumentarfilm ▁fra ... (+11 more)` | 21 |
+| 16k | `▁ 1 8 . ▁april ▁er ▁en ▁dansk ▁dokumentarfilm ▁fra ... (+11 more)` | 21 |
+| 32k | `▁ 1 8 . ▁april ▁er ▁en ▁dansk ▁dokumentarfilm ▁fra ... (+11 more)` | 21 |
+| 64k | `▁ 1 8 . ▁april ▁er ▁en ▁dansk ▁dokumentarfilm ▁fra ... (+11 more)` | 21 |
 
-...`
+**Sample 3:** `Takeshi Watanabe (født 10. september er en japansk fodboldspiller. Japans fodbol...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁se ▁også ▁ 5 9 8 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 16k | `▁se ▁også ▁ 5 9 8 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 32k | `▁se ▁også ▁ 5 9 8 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
-| 64k | `▁se ▁også ▁ 5 9 8 ▁( tal ) ▁begivenheder ... (+7 more)` | 17 |
+| 8k | `▁tak es hi ▁wat an ab e ▁( født ▁ ... (+12 more)` | 22 |
+| 16k | `▁tak es hi ▁wat an abe ▁( født ▁ 1 ... (+11 more)` | 21 |
+| 32k | `▁takes hi ▁wat an abe ▁( født ▁ 1 0 ... (+10 more)` | 20 |
+| 64k | `▁takes hi ▁watanabe ▁( født ▁ 1 0 . ▁september ... (+8 more)` | 18 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.256x compression
-- **Lowest UNK Rate:** 8k with 0.1175% unknown tokens
+- **Best Compression:** 64k achieves 4.557x compression
+- **Lowest UNK Rate:** 8k with 0.1227% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -143,57 +139,111 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 147,251 🏆 | 17.17 | 1,987,899 | 9.8% | 23.4% |
-| **2-gram** | 347 🏆 | 8.44 | 19,771 | 62.6% | 98.3% |
-| **3-gram** | 860,751 | 19.72 | 4,757,337 | 4.0% | 10.6% |
-| **3-gram** | 3,240 | 11.66 | 192,309 | 23.7% | 65.1% |
-| **4-gram** | 2,463,069 | 21.23 | 8,589,777 | 2.8% | 7.0% |
-| **4-gram** | 20,967 | 14.36 | 1,299,708 | 11.6% | 35.0% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 205,179 | 17.65 | 1,697,331 | 7.2% | 18.6% |
+| **2-gram** | Subword | 291 🏆 | 8.19 | 16,676 | 66.5% | 99.0% |
+| **3-gram** | Word | 930,238 | 19.83 | 3,294,874 | 2.7% | 7.8% |
+| **3-gram** | Subword | 2,629 | 11.36 | 143,880 | 25.6% | 68.8% |
+| **4-gram** | Word | 2,232,256 | 21.09 | 5,289,799 | 1.9% | 5.1% |
+| **4-gram** | Subword | 16,827 | 14.04 | 898,389 | 12.2% | 36.9% |
+| **5-gram** | Word | 1,710,284 | 20.71 | 3,467,271 | 1.9% | 5.4% |
+| **5-gram** | Subword | 78,315 | 16.26 | 3,371,746 | 6.1% | 20.8% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `er en` | 214,430 |
+| 2 | `eksterne henvisninger` | 158,401 |
+| 3 | `til at` | 148,332 |
+| 4 | `for at` | 127,680 |
+| 5 | `i den` | 98,315 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `referencer eksterne henvisninger` | 69,492 |
+| 2 | `eksterne henvisninger fra` | 52,148 |
+| 3 | `en del af` | 36,449 |
+| 4 | `fra danmark fra` | 31,038 |
+| 5 | `på grund af` | 24,747 |
+
+**4-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `referencer eksterne henvisninger fra` | 31,041 |
+| 2 | `fra danmark fra danmark` | 18,040 |
+| 3 | `eksterne henvisninger fra danmark` | 13,653 |
+| 4 | `eksterne henvisninger fra usa` | 10,607 |
+| 5 | `eksterne henvisninger film fra` | 8,857 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `referencer eksterne henvisninger fra danmark` | 8,198 |
+| 2 | `referencer eksterne henvisninger fra usa` | 7,710 |
+| 3 | `referencer eksterne henvisninger film fra` | 6,839 |
+| 4 | `fra danmark fra danmark fra` | 6,792 |
+| 5 | `eksterne henvisninger film fra fra` | 6,671 |
+
+**2-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `e r` | 14,686,017 |
+| 2 | `e _` | 12,413,595 |
+| 3 | `e n` | 11,692,715 |
+| 4 | `d e` | 11,106,628 |
+| 5 | `r _` | 9,958,657 |
+
+**3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `kategori :` | 797,923 |
-| 2 | `, og` | 388,695 |
-| 3 | `, der` | 367,530 |
-| 4 | `. i` | 304,754 |
-| 5 | `, som` | 301,325 |
+| 1 | `e r _` | 6,591,787 |
+| 2 | `e n _` | 5,761,673 |
+| 3 | `_ d e` | 4,088,356 |
+| 4 | `e t _` | 3,830,236 |
+| 5 | `_ i _` | 3,324,144 |
 
-**3-grams:**
+**4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `henvisninger kategori :` | 91,987 |
-| 2 | `eksterne henvisninger kategori` | 84,554 |
-| 3 | `danmark kategori :` | 75,266 |
-| 4 | `referencer eksterne henvisninger` | 69,753 |
-| 5 | `) er en` | 69,674 |
+| 1 | `_ o g _` | 2,559,398 |
+| 2 | `_ f o r` | 1,851,842 |
+| 3 | `_ a f _` | 1,698,296 |
+| 4 | `d e n _` | 1,598,615 |
+| 5 | `_ t i l` | 1,395,426 |
 
-**4-grams:**
+**5-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `eksterne henvisninger kategori :` | 84,554 |
-| 2 | `fra danmark kategori :` | 64,734 |
-| 3 | `referencer eksterne henvisninger kategori` | 49,030 |
-| 4 | `kategori : fodboldspillere fra` | 42,815 |
-| 5 | `bl . a .` | 42,727 |
+| 1 | `_ t i l _` | 1,111,382 |
+| 2 | `_ d e n _` | 1,013,475 |
+| 3 | `_ s o m _` | 926,452 |
+| 4 | `_ f r a _` | 883,398 |
+| 5 | `_ f o r _` | 860,091 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 347
+- **Best Perplexity:** 2-gram (subword) with 291
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~35% of corpus
+- **Coverage:** Top-1000 patterns cover ~21% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -201,55 +251,86 @@ Below are sample sentences tokenized with each vocabulary size:
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.7002 | 1.625 | 8.20 | 2,503,524 | 30.0% |
-| **1** | 1.4386 | 2.710 | 9.82 | 7,827 | 0.0% |
-| **2** | 0.4144 | 1.333 | 2.67 | 20,511,480 | 58.6% |
-| **2** | 0.7435 | 1.674 | 5.32 | 76,803 | 25.6% |
-| **3** | 0.1972 | 1.146 | 1.49 | 54,702,359 | 80.3% |
-| **3** | 0.9118 | 1.881 | 5.38 | 408,247 | 8.8% |
-| **4** | 0.0910 🏆 | 1.065 | 1.18 | 81,699,919 | 90.9% |
-| **4** | 0.7874 🏆 | 1.726 | 3.84 | 2,194,966 | 21.3% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 0.9282 | 1.903 | 11.03 | 2,011,765 | 7.2% |
+| **1** | Subword | 1.1734 | 2.255 | 7.58 | 8,958 | 0.0% |
+| **2** | Word | 0.3698 | 1.292 | 2.34 | 22,156,805 | 63.0% |
+| **2** | Subword | 0.6816 | 1.604 | 4.74 | 67,792 | 31.8% |
+| **3** | Word | 0.1562 | 1.114 | 1.36 | 51,659,329 | 84.4% |
+| **3** | Subword | 0.7837 | 1.722 | 4.70 | 321,234 | 21.6% |
+| **4** | Word | 0.0627 🏆 | 1.044 | 1.11 | 69,884,622 | 93.7% |
+| **4** | Subword | 0.7511 | 1.683 | 3.88 | 1,508,279 | 24.9% |
+
+### Generated Text Samples (Word-based)
+
+Below are text samples generated from each word-based Markov chain model:
+
+**Context Size 1:**
+
+1. `i sverige bernadotte en tidligere premierminister vladimír vašíček tjekkisk eller med langt de flest...`
+2. `og kortlagte dertil uhensigtsmæssige reaktionsmønstre på denne slags rum som et individuelt hold fra...`
+3. `af det eneste gang i lælehe lunden og shimonoseki afstod unionen og sydlige ishav og egyptiske`
+
+**Context Size 2:**
+
+1. `er en aristokrat fra oneglia på en figur på fordi de manglede stadig konkrete beviser det objekts`
+2. `eksterne henvisninger fra nederlandene fra flandern og champagne fra reims til danmark og derpaa ble...`
+3. `til at åbne sine egne retoriske færdigheder selvom de ikke mangler det umiddelbares friskhed inspira...`
 
-### Generated Text Samples
+**Context Size 3:**
+
+1. `referencer eksterne henvisninger 05 i vejle i alt var omkring 100 000 lysår og en tykkelse af cirka`
+2. `eksterne henvisninger fra mozambique fra maputo ved sommer ol mestre fra usa sølvmedaljevindere fra ...`
+3. `en del af moskenes kommune i nordland fylke i norge med et underskud på godt én million kroner`
+
+**Context Size 4:**
 
-Below are text samples generated from each Markov chain model:
+1. `referencer eksterne henvisninger fra storbritannien medaljevindere i gymnastik mestre fra grækenland...`
+2. `fra danmark fra danmark af videnskabernes selskab i dansk biografisk leksikon fra danmark thomas 1 f...`
+3. `eksterne henvisninger fra danmark film fra fra nordisk film dramafilm fra danmark instrueret af augu...`
+
+
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `. ali var således at lukke alle ved nogle indfødte var de fleste insektarter fundet sted`
-2. `, fra 1979 fuldmægtig ved opførelsen af en metrostation nedenunder , men palle von gyldenskiold (`
-3. `i fjendskab mellem køge bugt , som ejendommen var blevet beskrevet som har stenz . von`
+1. `_het_i_ldshic._k`
+2. `enoge_der_8.a_t.`
+3. `rerliolin,_opon_`
 
 **Context Size 2:**
 
-1. `kategori : arveret kategori : sovjetiske film fra 2009 kategori : museer i oslo kategori : modtagere`
-2. `, og det sydlige hellas og bliver dræbt af en anden organisation , der stod udenfor eu`
-3. `, der indsamler penge til sin karriere og er det dominerende træ . gunnar carlquist ( født`
+1. `er_ers_kum._ten_e`
+2. `e_asterdyra_et_fo`
+3. `en_i_kitler_være_`
 
 **Context Size 3:**
 
-1. `henvisninger kategori : ideologier kategori : etnicitet kategori : immaterialret kategori : straffel...`
-2. `eksterne henvisninger kategori : kenyas provinser`
-3. `danmark kategori : fløjtenister fra danmark kategori : maskinfabrikker i danmark kategori : det dans...`
+1. `er_randsbog_blev_p`
+2. `en_i_han_ver_guldv`
+3. `_det_af_daktat_og_`
 
 **Context Size 4:**
 
-1. `eksterne henvisninger kategori : fugle fra vestasien kategori : ræve`
-2. `fra danmark kategori : personer i dansk biografisk leksikon kategori : tyskere i 1900 - tallet kateg...`
-3. `referencer eksterne henvisninger kategori : elvis presley - sange kategori : singler fra 1963 katego...`
+1. `_og_kristia_schlesw`
+2. `_forfattish_music_d`
+3. `_af_storia_italiste`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 90.9% predictability
+- **Best Predictability:** Context-4 (word) with 93.7% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (2,194,966 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (1,508,279 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -265,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 974,563 |
-| Total Tokens | 94,217,021 |
-| Mean Frequency | 96.68 |
+| Vocabulary Size | 885,946 |
+| Total Tokens | 86,775,295 |
+| Mean Frequency | 97.95 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 6265.69 |
+| Frequency Std Dev | 6460.78 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | i | 3,414,957 |
-| 2 | og | 2,580,527 |
-| 3 | af | 1,724,110 |
-| 4 | en | 1,367,373 |
-| 5 | til | 1,141,339 |
-| 6 | er | 1,091,325 |
-| 7 | den | 1,045,804 |
-| 8 | at | 983,539 |
-| 9 | på | 952,118 |
-| 10 | som | 942,026 |
+| 1 | i | 3,396,891 |
+| 2 | og | 2,568,581 |
+| 3 | af | 1,716,528 |
+| 4 | en | 1,361,402 |
+| 5 | til | 1,134,702 |
+| 6 | er | 1,086,363 |
+| 7 | den | 1,040,601 |
+| 8 | at | 980,457 |
+| 9 | på | 948,450 |
+| 10 | som | 939,070 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | sinoefloden | 2 |
-| 2 | deathconsciousness | 2 |
-| 3 | folkedanseforeninger | 2 |
-| 4 | affranchi | 2 |
-| 5 | superfilmen | 2 |
-| 6 | kettletoft | 2 |
-| 7 | sandays | 2 |
-| 8 | crummack | 2 |
-| 9 | rousays | 2 |
-| 10 | 2025919140 | 2 |
+| 1 | elektronikinteresserede | 2 |
+| 2 | sinoefloden | 2 |
+| 3 | deathconsciousness | 2 |
+| 4 | folkedanseforeninger | 2 |
+| 5 | affranchi | 2 |
+| 6 | superfilmen | 2 |
+| 7 | kettletoft | 2 |
+| 8 | sandays | 2 |
+| 9 | crummack | 2 |
+| 10 | rousays | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0148 |
-| R² (Goodness of Fit) | 0.997060 |
+| Zipf Coefficient | 1.0001 |
+| R² (Goodness of Fit) | 0.998027 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 36.6% |
-| Top 1,000 | 57.4% |
+| Top 100 | 38.2% |
+| Top 1,000 | 58.1% |
 | Top 5,000 | 73.3% |
-| Top 10,000 | 79.6% |
+| Top 10,000 | 79.5% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9971 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 36.6% of corpus
-- **Long Tail:** 964,563 words needed for remaining 20.4% coverage
+- **Zipf Compliance:** R²=0.9980 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 38.2% of corpus
+- **Long Tail:** 875,946 words needed for remaining 20.5% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -335,24 +416,119 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 679,813 | 32 | 3.066 | 0.860 | 0.7852 🏆 |
-| **mono_64d** | 679,813 | 64 | 3.485 | 0.868 | 0.7644 |
-| **mono_128d** | 679,813 | 128 | 3.895 | 0.896 | 0.7063 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.7924 🏆 | 0.3816 | N/A | N/A |
+| **mono_64d** | 64 | 0.7720 | 0.3058 | N/A | N/A |
+| **mono_128d** | 128 | 0.7142 | 0.2314 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7924 | 0.3910 | 0.4140 | 0.7940 |
+| **aligned_64d** | 64 | 0.7720 | 0.3076 | 0.6360 | 0.9000 |
+| **aligned_128d** | 128 | 0.7142 | 0.2447 | 0.7560 | 0.9480 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7852 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 679,813 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** mono_32d with 0.7924 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3104. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 75.6% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.739** | Low formulaic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-e` | uforfalskede, ledocarpaceae, hærgende |
+| `-n` | fjerntogsperron, flexlinjen, industriudstillingen |
+| `-s` | bialiks, epicurus, ratios |
+| `-r` | bredevandsbakker, provinshertugdømmer, linseskyer |
+| `-er` | bredevandsbakker, provinshertugdømmer, linseskyer |
+| `-en` | flexlinjen, industriudstillingen, jordbundslæren |
+| `-et` | affrikeret, polyarkiet, panserkorpset |
+| `-ne` | heatene, beslutningsevne, skillingsviserne |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `irke` | 2.09x | 181 contexts | birke, virke, dirke |
+| `elig` | 1.65x | 256 contexts | helig, selig, zelig |
+| `embe` | 2.00x | 89 contexts | tembe, rembe, embed |
+| `nger` | 1.45x | 439 contexts | inger, enger, anger |
+| `tisk` | 1.73x | 152 contexts | tiske, etisk, tiski |
+| `ndel` | 1.42x | 393 contexts | andel, endel, ndele |
+| `mber` | 1.52x | 264 contexts | imber, amber, ember |
+| `nmar` | 1.77x | 85 contexts | anmary, enmark, donmar |
+| `lsen` | 1.52x | 174 contexts | elsen, ólsen, olsen |
+| `rste` | 1.33x | 307 contexts | erste, første, fyrste |
+| `rden` | 1.38x | 227 contexts | erden, urden, arden |
+| `oner` | 1.34x | 260 contexts | zoner, joner, loner |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+*No significant affix co-occurrences detected.*
+
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| profeterne | **`prof-et-er-ne`** | 7.5 | `prof` |
+| regierende | **`regi-er-en-de`** | 7.5 | `regi` |
+| kunstkritikeres | **`kunstkritik-er-es`** | 6.0 | `kunstkritik` |
+| buccaneer | **`bucca-ne-er`** | 6.0 | `bucca` |
+| udvikleres | **`udvikl-er-es`** | 6.0 | `udvikl` |
+| håndredskaberne | **`håndredskab-er-ne`** | 6.0 | `håndredskab` |
+| bolværkerne | **`bolværk-er-ne`** | 6.0 | `bolværk` |
+| autogenereret | **`autog-en-er-er-et`** | 6.0 | `autog` |
+| fællesgraven | **`fællesgrav-en`** | 4.5 | `fællesgrav` |
+| feltflyvepladser | **`feltflyveplads-er`** | 4.5 | `feltflyveplads` |
+| sangtrioen | **`sangtrio-en`** | 4.5 | `sangtrio` |
+| teknologiparken | **`teknologipark-en`** | 4.5 | `teknologipark` |
+| finnmarken | **`finnmark-en`** | 4.5 | `finnmark` |
+| patriarker | **`patriark-er`** | 4.5 | `patriark` |
+| synonymordbogen | **`synonymordbog-en`** | 4.5 | `synonymordbog` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Danish shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -360,11 +536,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (4.26x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (347) |
-| Markov | **Context-4** | Highest predictability (90.9%) |
+| Tokenizer | **64k BPE** | Best compression (4.56x) |
+| N-gram | **2-gram** | Lowest perplexity (291) |
+| Markov | **Context-4** | Highest predictability (93.7%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -554,7 +731,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -570,7 +748,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-29 09:01:56*
+*Report Date: 2026-01-08 09:40:43*

models/embeddings/aligned/da_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73ddba4319b9c61ec922776d5b343a9cbea1227c8c0767cf44417e5423ef9a9d
+size 1688091050

models/embeddings/aligned/da_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "da", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/da_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e960c2d1b89a6939207abf9f7febbf74e2003b55bd6f7adc3aecb53cf800ac82
+size 65664

models/embeddings/aligned/da_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "da",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 190290,
+  "vocab_size": 636667
+}

models/embeddings/aligned/da_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:988238892948f6febe998defae727df9b27ef13ad654566c2592db6e855c50e0
+size 431130794

models/embeddings/aligned/da_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "da", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/da_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4723793b1c4d686f28210200dba84730468f082b6ec836d2a334c3ae54e18fe
+size 4224

models/embeddings/aligned/da_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "da",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 190290,
+  "vocab_size": 636667
+}

models/embeddings/aligned/da_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc55665d9ef21bcf1bf7a7066f940953c4109b2066b330a89a85d89a420f561
+size 850117546

models/embeddings/aligned/da_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "da", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/da_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d144744dc4881b4e914e7dd36389eaa37118c43fd10a5dd6086f60d2700be0
+size 16512

models/embeddings/aligned/da_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "da",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 190290,
+  "vocab_size": 636667
+}

models/embeddings/monolingual/da_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68c3acea0b3acc83f64903aac63268a5886819a3724537b20264e8742bbe0389
-size 1733202257
+oid sha256:73ddba4319b9c61ec922776d5b343a9cbea1227c8c0767cf44417e5423ef9a9d
+size 1688091050

models/embeddings/monolingual/da_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 679813
+  "vocab_size": 636667
 }

models/embeddings/monolingual/da_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de1aa27bfa4123e9f8a42dedeb8f7bc963d4dce71c567da730cd9d71333333e2
-size 443105873
+oid sha256:988238892948f6febe998defae727df9b27ef13ad654566c2592db6e855c50e0
+size 431130794

models/embeddings/monolingual/da_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 679813
+  "vocab_size": 636667
 }

models/embeddings/monolingual/da_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:465fb7807ee1c1b7cc0cbc92c8b94765964d007aaca0e89095d66ba847a12af5
-size 873138001
+oid sha256:fbc55665d9ef21bcf1bf7a7066f940953c4109b2066b330a89a85d89a420f561
+size 850117546

models/embeddings/monolingual/da_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 679813
+  "vocab_size": 636667
 }

models/subword_markov/da_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36e2282c95edc33f735242830727d3cb38b457f0d60410f8fc958ae54a853cee
-size 501415
+oid sha256:a77f7bbbed8bb0a1b7ca11c531f2eee1c420a32a038d5bf767d0f10d93792553
+size 472603

models/subword_markov/da_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "da",
-  "unique_contexts": 7827,
-  "total_transitions": 630161089
+  "unique_contexts": 8958,
+  "total_transitions": 567528445
 }

models/subword_markov/da_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f3e3b34698c83d39fb848b9ed450e9d333640fb38b7af701911a7bf5b6c4470
-size 3244857
+oid sha256:02e021439972f73ee932e292c346b43df0b703b3962578d83df42a1d52b05ad8
+size 2695498

models/subword_markov/da_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "da",
-  "unique_contexts": 76803,
-  "total_transitions": 629849363
+  "unique_contexts": 67792,
+  "total_transitions": 567217531
 }

models/subword_markov/da_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1a5503d267e8fad2c6f3d8d57b043b8b39f15768b2399cf0cf521aba952aa5d
-size 16479657
+oid sha256:8b242be7ec2b5a684749ddaa6ebad236c5e339177310a6202976c8283f1acb95
+size 12404724

models/subword_markov/da_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "da",
-  "unique_contexts": 408247,
-  "total_transitions": 629537637
+  "unique_contexts": 321234,
+  "total_transitions": 566906617
 }

models/subword_markov/da_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07015bd1fa896217f9ece588bbcfab316cebe27c1f5353d7bd0afb2b0f4effb4
-size 64259074
+oid sha256:a7e3286db99aec5d2b702be54d85f0a66d05ce5fc24b6fa20ff723b07e868835
+size 45985973

models/subword_markov/da_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "da",
-  "unique_contexts": 2194966,
-  "total_transitions": 629225911
+  "unique_contexts": 1508279,
+  "total_transitions": 566595703
 }

models/subword_ngram/da_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8ca2619f57b1ac7bbefcabc250a3418b8e0a026afbf506eb2d9ec32d51d4dc8
-size 273435
+oid sha256:bd36bc8cbd9bf04043fe587c2e91b0166f2f902ab648df930bf75a9be95bcc8e
+size 232894

models/subword_ngram/da_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "da",
-  "unique_ngrams": 19771,
-  "total_ngrams": 630161089
+  "unique_ngrams": 16676,
+  "total_ngrams": 567528445
 }

models/subword_ngram/da_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14e01ea2fde43d22f7708e4abd070d10512956496ee36e85a20a8a92ce15358f
-size 2327975
+oid sha256:106a616b5c1b46994fe3713bf07964e194dc7024d1343aa576e1f55ffee5566c
+size 1806805

models/subword_ngram/da_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "da",
-  "unique_ngrams": 192309,
-  "total_ngrams": 629849363
+  "unique_ngrams": 143880,
+  "total_ngrams": 567217531
 }

models/subword_ngram/da_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13389e44c39a59c594db7e4f570fdfddecb9c53507210fcd712e5a2e80816e26
-size 15313345
+oid sha256:a8ad24dc194e28d56de6bd6b6752b313f4078f256958653b23bfcaf4f18384a6
+size 10697338

models/subword_ngram/da_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "da",
-  "unique_ngrams": 1299708,
-  "total_ngrams": 629537637
+  "unique_ngrams": 898389,
+  "total_ngrams": 566906617
 }

models/subword_ngram/da_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c69c03af9234e68acd0c9634d7d828649f0dbb28f5225828f05240edb0b01a
+size 40665412

models/subword_ngram/da_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "da",
+  "unique_ngrams": 3371746,
+  "total_ngrams": 566595703
+}

models/tokenizer/da_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5257f8c08d312560577f764125bf86dee3ad20060303d5d179efc751e6926aaa
-size 509881
+oid sha256:b6c8bf9b62ee8ed799c0d6e5bebc50005a083cc2be10b93cd181bd4a4436bd9f
+size 510050

models/tokenizer/da_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72f637e73e0c17b40cdde4b080377b234f3d7ddb88a0836e27d0124aec823468
-size 792817
+oid sha256:324d7d968f80c82e64fdbdac1d11337ffbd42016efa24bdc474f7c51a18f0d34
+size 793243

models/tokenizer/da_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f34a8719a8206abf56c31b11ccd067be63136835dddf9ee6648468ac9b749ed
-size 1376877
+oid sha256:09328d0a93713ba20c7759335de10e464b5b3fbba78c63e9902797a21f2e0c25
+size 1376208

models/tokenizer/da_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a9341fa8df03ff9f7d47d5875e47073097d21830ea7a2b2b07ddda14141fb9e
-size 372662
+oid sha256:0548659fdbbe7d6f4691b828e7ac15b89cfdc9c2b7b45ce559d3fa9f991cc1ad
+size 372983

models/vocabulary/da_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5eaf29d24437f2bdfcf35e6f6e0f105f1b298b3da7b0c1e2899592ab05ed482
-size 15264314
+oid sha256:33214b86b6b63aa31bc1355ba9be1ff25be7945088ca5b0916ab22f1725454af
+size 13968436

models/vocabulary/da_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "da",
-  "vocabulary_size": 974563,
+  "vocabulary_size": 885946,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.02614497849971959,
+    "type_token_ratio": 0.022906069788993663,
     "coverage": {
-      "top_100": 0.36016111340236506,
-      "top_1000": 0.5648239365833638,
-      "top_5000": 0.7214206987908641,
-      "top_10000": 0.7832362713874816
+      "top_100": 0.3767927431893056,
+      "top_1000": 0.5733930744322329,
+      "top_5000": 0.7236143220735781,
+      "top_10000": 0.7852248444527253
     },
-    "hapax_count": 1528707,
-    "hapax_ratio": 0.6106840252949143,
-    "total_documents": 311726
+    "hapax_count": 1127563,
+    "hapax_ratio": 0.5599989868433665,
+    "total_documents": 310914
   }
 }

models/word_markov/da_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd504f360cbe5a90b1c7b013522b8db902e17e2e16ba3ba9c36f60150cd26922
-size 194239107
+oid sha256:c576188f3cd603eafa88b134c0d4863d36f5b6e3fd8ed58400bcdb5359e10454
+size 203010577

models/word_markov/da_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "da",
-  "unique_contexts": 2503524,
-  "total_transitions": 118622822
+  "unique_contexts": 2011765,
+  "total_transitions": 87591944
 }

models/word_markov/da_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:594706495f332b8802a7647db03b3acd0fc22282c1fbc02ec68f34b4f84095d8
-size 631690789
+oid sha256:695681b3d74d7d99c09b4665ac5ada28f4f214399b9a8185ef678510f9d563bd
+size 638125991

models/word_markov/da_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "da",
-  "unique_contexts": 20511480,
-  "total_transitions": 118311096
+  "unique_contexts": 22156805,
+  "total_transitions": 87281030
 }