{
 "run_at": "2026-10-12T16:53:56+00:00",
 "python": "3.13.16",
 "platform": "Linux-6.18.44-fc-v114-x86_64-with-glibc2.39",
 "packages": {
  "tiktoken": "0.14.0",
  "tokenizers": "0.23.3",
  "transformers": "5.19.0",
  "huggingface_hub": "2.2.0",
  "datasets": "5.1.0",
  "sentencepiece": "0.2.2",
  "protobuf": "7.36.1"
 },
 "method": {
  "special_tokens": "none (tiktoken encode_ordinary; HF add_special_tokens=False)",
  "unit": "each sentence encoded separately; totals summed",
  "normalisation": "Unicode NFC, leading/trailing whitespace stripped",
  "chars": "Unicode code points",
  "words": "str.split() on whitespace",
  "ratio_vs_en": "tokens(lang) / tokens(en), same tokenizer and dataset",
  "public_set_size": 200
 },
 "datasets": [
  {
   "name": "school",
   "source": "texts/school.json",
   "licence": "CC BY 4.0 (Bodham Labs, original)"
  }
 ],
 "skipped": [
  {
   "name": "openlanguagedata/flores_plus",
   "kind": "dataset",
   "error": "gated repo on the HF Hub; no HF access token in this environment. GatedRepoError: 401 Client Error. (Request ID: Root=1-6acd109c-2ba4570a57b625fa3992f44a;fc8b26b9-c2b4-4c12-af0e-46a0a9fc966c)"
  },
  {
   "name": "ai4bharat/IN22-Gen",
   "kind": "dataset",
   "error": "gated repo on the HF Hub; no HF access token in this environment. GatedRepoError: 401 Client Error. (Request ID: Root=1-6acd109c-70cfda3b4a248c3870abb059;42775a68-ff76-49da-9297-5ad1a2f8724f)"
  },
  {
   "name": "google/gemma-3-270m",
   "kind": "tokenizer",
   "error": "gated repo on the HF Hub; no HF access token in this environment. GatedRepoError: 401 Client Error. (Request ID: Root=1-6acd10a0-689f3be5188dd42e48e2850e;b7ff130e-c2e2-48b3-ab59-28dab6aef449)"
  },
  {
   "name": "meta-llama/Llama-3.2-1B",
   "kind": "tokenizer",
   "error": "gated repo on the HF Hub; no HF access token in this environment. GatedRepoError: 401 Client Error. (Request ID: Root=1-6acd10a1-7673471a1baee5d16cdc9181;b035914f-8461-48a8-9a4f-b353b88f838f)"
  }
 ],
 "crosscheck": [
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 204,
   "tokens_alt": 204,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 929,
   "tokens_alt": 929,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 329,
   "tokens_alt": 329,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 623,
   "tokens_alt": 623,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 204,
   "tokens_alt": 204,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 929,
   "tokens_alt": 929,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 329,
   "tokens_alt": 329,
   "match": true
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 623,
   "tokens_alt": 623,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 215,
   "tokens_alt": 215,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 1042,
   "tokens_alt": 1042,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 361,
   "tokens_alt": 361,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 719,
   "tokens_alt": 719,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 215,
   "tokens_alt": 215,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 1042,
   "tokens_alt": 1042,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 361,
   "tokens_alt": 361,
   "match": true
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 719,
   "tokens_alt": 719,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 211,
   "tokens_alt": 211,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 274,
   "tokens_alt": 274,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 406,
   "tokens_alt": 406,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 248,
   "tokens_alt": 248,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 211,
   "tokens_alt": 211,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 274,
   "tokens_alt": 274,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 406,
   "tokens_alt": 406,
   "match": true
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "impl": "sentencepiece(tokenizer.model)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 248,
   "tokens_alt": 248,
   "match": true
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "impl": "sentencepiece(spiece.model)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 257,
   "tokens_alt": 257,
   "match": true
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "impl": "sentencepiece(spiece.model)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 291,
   "tokens_alt": 291,
   "match": true
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "impl": "sentencepiece(spiece.model)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 446,
   "tokens_alt": 446,
   "match": true
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "impl": "sentencepiece(spiece.model)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 296,
   "tokens_alt": 296,
   "match": true
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "en",
   "tokens_main": 211,
   "tokens_alt": 211,
   "match": true
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi",
   "tokens_main": 411,
   "tokens_alt": 411,
   "match": true
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hinglish",
   "tokens_main": 317,
   "tokens_alt": 317,
   "match": true
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "impl": "tokenizers(tokenizer.json)",
   "dataset": "school",
   "lang": "hi_mixed",
   "tokens_main": 293,
   "tokens_alt": 293,
   "match": true
  }
 ],
 "rows": [
  {
   "tokenizer": "tiktoken/o200k_base",
   "tokenizer_version": "tiktoken 0.14.0 o200k_base (n_vocab=200019)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 199,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2047,
   "tokens_per_word": 1.157,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/o200k_base",
   "tokenizer_version": "tiktoken 0.14.0 o200k_base (n_vocab=200019)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 321,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.3242,
   "tokens_per_word": 1.597,
   "ratio_vs_en": 1.6131,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/o200k_base",
   "tokenizer_version": "tiktoken 0.14.0 o200k_base (n_vocab=200019)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 295,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.2601,
   "tokens_per_word": 1.4824,
   "ratio_vs_en": 1.4824,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/o200k_base",
   "tokenizer_version": "tiktoken 0.14.0 o200k_base (n_vocab=200019)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 249,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.2422,
   "tokens_per_word": 1.2513,
   "ratio_vs_en": 1.2513,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/cl100k_base",
   "tokenizer_version": "tiktoken 0.14.0 cl100k_base (n_vocab=100277)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 202,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2078,
   "tokens_per_word": 1.1744,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/cl100k_base",
   "tokenizer_version": "tiktoken 0.14.0 cl100k_base (n_vocab=100277)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 991,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 1.001,
   "tokens_per_word": 4.9303,
   "ratio_vs_en": 4.9059,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/cl100k_base",
   "tokenizer_version": "tiktoken 0.14.0 cl100k_base (n_vocab=100277)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 327,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.2884,
   "tokens_per_word": 1.6432,
   "ratio_vs_en": 1.6188,
   "unk_tokens": 0
  },
  {
   "tokenizer": "tiktoken/cl100k_base",
   "tokenizer_version": "tiktoken 0.14.0 cl100k_base (n_vocab=100277)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 657,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.6391,
   "tokens_per_word": 3.3015,
   "ratio_vs_en": 3.2525,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "tokenizer_version": "Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf (Qwen2Tokenizer, vocab=151665)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 204,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2099,
   "tokens_per_word": 1.186,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "tokenizer_version": "Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf (Qwen2Tokenizer, vocab=151665)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 929,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.9384,
   "tokens_per_word": 4.6219,
   "ratio_vs_en": 4.5539,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "tokenizer_version": "Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf (Qwen2Tokenizer, vocab=151665)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 329,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.2901,
   "tokens_per_word": 1.6533,
   "ratio_vs_en": 1.6127,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen2.5-0.5B-Instruct",
   "tokenizer_version": "Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf (Qwen2Tokenizer, vocab=151665)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 623,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.606,
   "tokens_per_word": 3.1307,
   "ratio_vs_en": 3.0539,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "tokenizer_version": "Qwen/Qwen3-0.6B@c1899de289a0 (Qwen2Tokenizer, vocab=151669)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 204,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2099,
   "tokens_per_word": 1.186,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "tokenizer_version": "Qwen/Qwen3-0.6B@c1899de289a0 (Qwen2Tokenizer, vocab=151669)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 929,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.9384,
   "tokens_per_word": 4.6219,
   "ratio_vs_en": 4.5539,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "tokenizer_version": "Qwen/Qwen3-0.6B@c1899de289a0 (Qwen2Tokenizer, vocab=151669)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 329,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.2901,
   "tokens_per_word": 1.6533,
   "ratio_vs_en": 1.6127,
   "unk_tokens": 0
  },
  {
   "tokenizer": "Qwen/Qwen3-0.6B",
   "tokenizer_version": "Qwen/Qwen3-0.6B@c1899de289a0 (Qwen2Tokenizer, vocab=151669)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 623,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.606,
   "tokens_per_word": 3.1307,
   "ratio_vs_en": 3.0539,
   "unk_tokens": 0
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "tokenizer_version": "mistralai/Mistral-7B-v0.1@27d67f1b5f57 (TokenizersBackend, vocab=32000)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 215,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2212,
   "tokens_per_word": 1.25,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "tokenizer_version": "mistralai/Mistral-7B-v0.1@27d67f1b5f57 (TokenizersBackend, vocab=32000)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 1042,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 1.0525,
   "tokens_per_word": 5.1841,
   "ratio_vs_en": 4.8465,
   "unk_tokens": 0
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "tokenizer_version": "mistralai/Mistral-7B-v0.1@27d67f1b5f57 (TokenizersBackend, vocab=32000)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 361,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.3183,
   "tokens_per_word": 1.8141,
   "ratio_vs_en": 1.6791,
   "unk_tokens": 0
  },
  {
   "tokenizer": "mistralai/Mistral-7B-v0.1",
   "tokenizer_version": "mistralai/Mistral-7B-v0.1@27d67f1b5f57 (TokenizersBackend, vocab=32000)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 719,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.6994,
   "tokens_per_word": 3.6131,
   "ratio_vs_en": 3.3442,
   "unk_tokens": 0
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "tokenizer_version": "sarvamai/sarvam-1@e9607337286d (LlamaTokenizer, vocab=68096)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 211,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2171,
   "tokens_per_word": 1.2267,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "tokenizer_version": "sarvamai/sarvam-1@e9607337286d (LlamaTokenizer, vocab=68096)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 274,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.2768,
   "tokens_per_word": 1.3632,
   "ratio_vs_en": 1.2986,
   "unk_tokens": 0
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "tokenizer_version": "sarvamai/sarvam-1@e9607337286d (LlamaTokenizer, vocab=68096)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 406,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.358,
   "tokens_per_word": 2.0402,
   "ratio_vs_en": 1.9242,
   "unk_tokens": 0
  },
  {
   "tokenizer": "sarvamai/sarvam-1",
   "tokenizer_version": "sarvamai/sarvam-1@e9607337286d (LlamaTokenizer, vocab=68096)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 248,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.2412,
   "tokens_per_word": 1.2462,
   "ratio_vs_en": 1.1754,
   "unk_tokens": 0
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "tokenizer_version": "ai4bharat/IndicBARTSS@4b2669d25bc2 (AlbertTokenizer, vocab=64015)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 257,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2644,
   "tokens_per_word": 1.4942,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "tokenizer_version": "ai4bharat/IndicBARTSS@4b2669d25bc2 (AlbertTokenizer, vocab=64015)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 291,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.2939,
   "tokens_per_word": 1.4478,
   "ratio_vs_en": 1.1323,
   "unk_tokens": 0
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "tokenizer_version": "ai4bharat/IndicBARTSS@4b2669d25bc2 (AlbertTokenizer, vocab=64015)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 446,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.3933,
   "tokens_per_word": 2.2412,
   "ratio_vs_en": 1.7354,
   "unk_tokens": 0
  },
  {
   "tokenizer": "ai4bharat/IndicBARTSS",
   "tokenizer_version": "ai4bharat/IndicBARTSS@4b2669d25bc2 (AlbertTokenizer, vocab=64015)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 296,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.2879,
   "tokens_per_word": 1.4874,
   "ratio_vs_en": 1.1518,
   "unk_tokens": 0
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "tokenizer_version": "bert-base-multilingual-cased@3f076fdb1ab6 (BertTokenizer, vocab=119547)",
   "dataset": "school",
   "lang": "en",
   "n_sentences": 12,
   "tokens": 211,
   "chars": 972,
   "words": 172,
   "tokens_per_char": 0.2171,
   "tokens_per_word": 1.2267,
   "ratio_vs_en": 1.0,
   "unk_tokens": 0
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "tokenizer_version": "bert-base-multilingual-cased@3f076fdb1ab6 (BertTokenizer, vocab=119547)",
   "dataset": "school",
   "lang": "hi",
   "n_sentences": 12,
   "tokens": 411,
   "chars": 990,
   "words": 201,
   "tokens_per_char": 0.4152,
   "tokens_per_word": 2.0448,
   "ratio_vs_en": 1.9479,
   "unk_tokens": 0
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "tokenizer_version": "bert-base-multilingual-cased@3f076fdb1ab6 (BertTokenizer, vocab=119547)",
   "dataset": "school",
   "lang": "hinglish",
   "n_sentences": 12,
   "tokens": 317,
   "chars": 1134,
   "words": 199,
   "tokens_per_char": 0.2795,
   "tokens_per_word": 1.593,
   "ratio_vs_en": 1.5024,
   "unk_tokens": 0
  },
  {
   "tokenizer": "bert-base-multilingual-cased",
   "tokenizer_version": "bert-base-multilingual-cased@3f076fdb1ab6 (BertTokenizer, vocab=119547)",
   "dataset": "school",
   "lang": "hi_mixed",
   "n_sentences": 12,
   "tokens": 293,
   "chars": 1028,
   "words": 199,
   "tokens_per_char": 0.285,
   "tokens_per_word": 1.4724,
   "ratio_vs_en": 1.3886,
   "unk_tokens": 0
  }
 ]
}