Halaman utama Jelajahi Bantuan
Masuk
patrick
/
Trixy7
1
0
Fork 0
Berkas Masalah 0 Tarik Permintaan 0 Wiki
Cabang: master
Ranting Tag
Trixy7
/
models
/
intent
/
tokenizer
/
tokenizer_config.json

tokenizer_config.json 1.4 KB
Permalink Riwayat Mentahan

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465 
  1. {
    2. 

  2.   "added_tokens_decoder": {
    3. 

  3.     "0": {
    4. 

  4.       "content": "<s>",
    5. 

  5.       "lstrip": false,
    6. 

  6.       "normalized": false,
    7. 

  7.       "rstrip": false,
    8. 

  8.       "single_word": false,
    9. 

  9.       "special": true
    10. 

  10.     },
    11. 

  11.     "1": {
    12. 

  12.       "content": "<pad>",
    13. 

  13.       "lstrip": false,
    14. 

  14.       "normalized": false,
    15. 

  15.       "rstrip": false,
    16. 

  16.       "single_word": false,
    17. 

  17.       "special": true
    18. 

  18.     },
    19. 

  19.     "2": {
    20. 

  20.       "content": "</s>",
    21. 

  21.       "lstrip": false,
    22. 

  22.       "normalized": false,
    23. 

  23.       "rstrip": false,
    24. 

  24.       "single_word": false,
    25. 

  25.       "special": true
    26. 

  26.     },
    27. 

  27.     "3": {
    28. 

  28.       "content": "<unk>",
    29. 

  29.       "lstrip": false,
    30. 

  30.       "normalized": false,
    31. 

  31.       "rstrip": false,
    32. 

  32.       "single_word": false,
    33. 

  33.       "special": true
    34. 

  34.     },
    35. 

  35.     "250001": {
    36. 

  36.       "content": "<mask>",
    37. 

  37.       "lstrip": true,
    38. 

  38.       "normalized": false,
    39. 

  39.       "rstrip": false,
    40. 

  40.       "single_word": false,
    41. 

  41.       "special": true
    42. 

  42.     }
    43. 

  43.   },
    44. 

  44.   "bos_token": "<s>",
    45. 

  45.   "clean_up_tokenization_spaces": false,
    46. 

  46.   "cls_token": "<s>",
    47. 

  47.   "do_lower_case": true,
    48. 

  48.   "eos_token": "</s>",
    49. 

  49.   "extra_special_tokens": {},
    50. 

  50.   "mask_token": "<mask>",
    51. 

  51.   "max_length": 128,
    52. 

  52.   "model_max_length": 128,
    53. 

  53.   "pad_to_multiple_of": null,
    54. 

  54.   "pad_token": "<pad>",
    55. 

  55.   "pad_token_type_id": 0,
    56. 

  56.   "padding_side": "right",
    57. 

  57.   "sep_token": "</s>",
    58. 

  58.   "stride": 0,
    59. 

  59.   "strip_accents": null,
    60. 

  60.   "tokenize_chinese_chars": true,
    61. 

  61.   "tokenizer_class": "PreTrainedTokenizerFast",
    62. 

  62.   "truncation_side": "right",
    63. 

  63.   "truncation_strategy": "longest_first",
    64. 

  64.   "unk_token": "<unk>"
    65. 

  65. }
    66.