Commit Graph

Select branches

Hide Pull Requests

add-added-tokens-encoder

add-some-doc

add_decoders_to_doc

assign-token

austinleedavis/main

backlog_release

backtrack

bench-python

big_science_fork2

bigscience_fork

branch_0.13.4.rc2

branch_0.14.0

branch_0.15.0

branch_0.19.1

branch_0.20.1.rc1

branch_v0.14.0.rc1

branch_v0.15.1.rc1

branch_v0.15.2.rc1

branch_v0.16.0.rc1

branch_v0.20.0.rc1

cache-evmap

change_decoder_trait

ci-test-huggingface-hub-v1.0.0.rc0

clefourrier-patch-version-dep-hub

clippy_update

copilot/remove-stderr-warning-python-bindings

debug-release-ci

disable-par-iter

doc_build_typo

docs_new_frontend

extra-tokens

extra_wheel

fast-encode

fast-encode-2

fast-regex

feat/early_exit_right_truncation

feat/improve_whitespace_splitting_perf

feat/truncate_overflow_with_stride_test

feature/post-process-tokens

feature/role-to-token

feature/special-tokens-api

feature/unigram-unk-token

fix-cache-issues

fix-merge-words

fix-pretokenizer

fix-split-special

fix_ahash_pub

fix_anaconda

fix_benchmark2

fix_manylinux

fix_special_tokens_mask

fixing_decoder_bug_before_release

git_0.14.1

git_0.14.1rc1

git_0.15.1rc0

git_0.19.0

git_0.19.0rc0

git_v0.20.3-rc0

git_v0.20.4-rc0

git_v0.21.0

git_v0.21.1

git_v0.21.1_rc0

git_v0.21.3

git_v0.21.4

git_v0.22.0

impl_serde_type_struct

java-binding

main

metaspace_nosplit

mishig25-patch-1

mishig25-patch-2

model-config

modify_processor_trait

narsil/backtrack

patched

pep440

pickling_trainers

pin_docutils

prepare_patch

python-313-release

refactor_node

remove_unecessary_yelling

revert-1047-modify_processor_trait

serde_pyo3

skip_tokens_too_long_bpe

template_post_processor

test-old-decode

thomas/get_white_space_regex_with_byte_level

thomas/no_regex_byte_level

thomas/unigram_parallel_e_step

tmp

tmp_release

tmpbytefallback_hack

tryout

u64_branch

unigram_bug_maybe

update-serialize

update-stream

update-stuff

update_rand_and_clap_rebase

updating_replace_normalized_string_behavior

used_vendored_for_manylinux

v0.13.0-rc1

v0.20.3-release

v0.21.2

v0.21.2_rc0

v0.22.0-rc0

v0.22.1-rc0

v0.22.2-rc0

windows_aarch

wordpiece_aho

#1

#1005

#1008

#1009

#101

#1019

#102

#1021

#1024

#103

#1030

#1032

#1034

#1038

#1047

#1048

#1049

#1051

#1052

#1053

#1054

#1055

#1056

#1060

#1063

#1064

#1066

#1067

#1069

#107

#1072

#1079

#1081

#1082

#109

#1095

#1096

#1098

#1099

#110

#1101

#1102

#1107

#1108

#111

#1115

#1116

#112

#1120

#1123

#1124

#1125

#1126

#1127

#1129

#1130

#1131

#1133

#1134

#1136

#1137

#1139

#114

#1140

#1142

#1144

#1145

#1147

#1148

#1150

#1152

#1153

#1154

#1157

#116

#1166

#1167

#117

#1173

#118

#1181

#1182

#1183

#1188

#1189

#1191

#1192

#1194

#1195

#1196

#1197

#1199

#1203

#1205

#1207

#1209

#1209

#121

#1210

#1211

#1212

#1217

#1222

#1227

#1228

#1233

#1235

#124

#1244

#1245

#1246

#1248

#125

#1251

#1254

#1255

#1255

#1256

#1257

#126

#1262

#1264

#1266

#1268

#1270

#1271

#1272

#1273

#1275

#128

#1288

#1289

#129

#1291

#1292

#1293

#1294

#1295

#1296

#1299

#1302

#1303

#1306

#1308

#131

#1312

#1316

#1317

#1318

#1319

#132

#1320

#1321

#1322

#1328

#1329

#1330

#1331

#1333

#1335

#1338

#1339

#134

#1341

#1344

#1353

#1355

#1356

#1357

#136

#1367

#1370

#1371

#1377

#1378

#1379

#138

#1381

#1382

#1383

#1385

#1386

#1388

#1390

#1394

#1396

#140

#1404

#1406

#1413

#1414

#1419

#1420

#1428

#1430

#1433

#1435

#1436

#1437

#1438

#1441

#1442

#1443

#1444

#1453

#1456

#1459

#146

#1462

#1465

#147

#1472

#1476

#1478

#1479

#1482

#1483

#1484

#1486

#1488

#149

#1492

#1493

#1494

#1497

#1498

#1499

#1503

#1504

#1508

#1510

#1511

#1513

#1521

#1532

#1533

#1535

#1540

#1542

#1547

#1548

#155

#1550

#1551

#1554

#1555

#1557

#1558

#1560

#1561

#1568

#1569

#1570

#1570

#1571

#1574

#1580

#1582

#1583

#1584

#1585

#1586

#1587

#1588

#1589

#1590

#1591

#1592

#1593

#1594

#1595

#1597

#1598

#1599

#1600

#1602

#1603

#1604

#1605

#1605

#1606

#1607

#1608

#1610

#1617

#1618

#162

#1621

#1622

#1626

#1629

#1630

#1631

#1632

#1638

#1640

#1641

#1642

#1646

#1647

#1648

#165

#1651

#1652

#1653

#1659

#1660

#1662

#1664

#1665

#1667

#1669

#167

#1670

#1673

#1674

#1675

#1676

#1677

#1678

#1679

#1685

#1687

#1689

#169

#1693

#1699

#1701

#1707

#1708

#1709

#171

#1710

#1711

#1712

#1712

#1713

#1714

#1715

#1716

#1717

#1719

#1719

#172

#1724

#1725

#1732

#1733

#1733

#1737

#1739

#1744

#1745

#1746

#1747

#1750

#1752

#1753

#1753

#1755

#1755

#1756

#1758

#1762

#1763

#1763

#1764

#1766

#1770

#1771

#1772

#1774

#1778

#1780

#1781

#1782

#1783

#1783

#1785

#1788

#1788

#1789

#1789

#179

#1792

#1796

#1798

#1799

#18

#180

#1800

#1801

#1802

#1804

#1804

#1805

#1806

#1808

#1809

#1814

#1818

#1822

#1822

#1827

#183

#1830

#1830

#1831

#1832

#1832

#1834

#1838

#184

#1840

#1841

#1841

#1842

#1842

#1843

#1844

#1846

#1848

#1849

#1852

#1853

#1856

#1857

#1859

#1859

#1860

#1861

#1862

#1862

#1864

#1865

#1866

#1867

#1869

#1872

#1874

#1875

#1878

#188

#1880

#1880

#1883

#1884

#1888

#1888

#1889

#1891

#1892

#1893

#1893

#1894

#1894

#1895

#1895

#1896

#1898

#1899

#19

#190

#1901

#1903

#1905

#1907

#1911

#1912

#1912

#1916

#1916

#1921

#1923

#1924

#1925

#1928

#193

#1930

#1931

#1931

#1932

#1932

#1934

#1935

#1936

#1937

#1938

#1939

#1940

#1941

#1942

#1942

#1943

#1943

#1944

#1944

#1945

#1945

#1946

#1948

#1949

#1950

#1951

#1952

#1952

#1953

#1955

#1956

#1958

#1959

#1959

#196

#1963

#1964

#1965

#1965

#1966

#1967

#1967

#1968

#1968

#1969

#197

#1970

#1970

#1971

#1972

#1974

#1974

#1976

#1976

#1977

#1978

#1979

#1980

#1981

#1982

#1983

#1984

#1985

#1985

#1986

#1987

#1988

#1989

#1990

#1990

#2

#20

#200

#202

#206

#208

#21

#212

#213

#22

#220

#221

#222

#224

#226

#227

#229

#23

#234

#236

#238

#239

#24

#241

#249

#25

#26

#27

#271

#272

#273

#274

#276

#277

#28

#280

#286

#289

#29

#292

#298

#3

#303

#306

#308

#309

#311

#312

#315

#32

#327

#329

#33

#330

#333

#334

#338

#34

#344

#346

#347

#348

#350

#355

#356

#357

#36

#360

#362

#363

#364

#365

#366

#368

#37

#373

#377

#378

#379

#38

#383

#384

#385

#386

#387

#388

#389

#39

#390

#392

#393

#394

#395

#4

#40

#400

#401

#403

#407

#41

#416

#417

#42

#420

#421

#426

#427

#428

#429

#43

#430

#433

#434

#436

#44

#440

#441

#442

#445

#448

#45

#452

#453

#454

#455

#459

#462

#463

#464

#467

#47

#470

#473

#476

#477

#48

#480

#481

#485

#49

#490

#492

#494

#495

#498

#5

#50

#500

#503

#506

#508

#509

#512

#513

#514

#516

#518

#519

#520

#530

#533

#538

#539

#542

#543

#544

#549

#55

#551

#557

#559

#561

#565

#569

#573

#574

#582

#583

#589

#590

#591

#593

#594

#595

#596

#6

#60

#610

#616

#617

#618

#620

#621

#622

#625

#64

#650

#652

#653

#656

#657

#660

#671

#674

#675

#676

#68

#686

#691

#693

#7

#70

#700

#701

#702

#707

#708

#71

#713

#714

#716

#718

#719

#720

#721

#727

#728

#734

#743

#75

#758

#76

#762

#763

#764

#768

#77

#770

#771

#773

#774

#775

#78

#780

#789

#792

#793

#794

#795

#799

#8

#80

#806

#825

#831

#832

#833

#837

#841

#842

#844

#845

#846

#850

#851

#856

#857

#858

#859

#860

#863

#864

#868

#869

#870

#871

#877

#881

#882

#884

#887

#890

#894

#895

#896

#90

#901

#904

#907

#908

#909

#913

#915

#916

#919

#92

#920

#921

#923

#924

#925

#927

#93

#936

#937

#938

#939

#94

#940

#942

#943

#949

#95

#951

#952

#954

#956

#958

#960

#961

#962

#963

#966

#967

#969

#971

#973

#975

#976

#978

#980

#982

#986

#987

#988

#989

#99

#992

node-v0.12.0

node-v0.13.0

node-v0.13.1

node-v0.13.2

node-v0.13.3

node-v0.2.3

node-v0.2.4

node-v0.3.0

node-v0.3.1

node-v0.4.0

node-v0.4.1

node-v0.5.0

node-v0.6.0

node-v0.6.1

node-v0.6.2

node-v0.7.0

node-v0.8.0

node-v0.8.1

node-v0.8.2

node-v0.8.3

python-v0.10.0

python-v0.10.0rc1

python-v0.10.1

python-v0.10.1rc1

python-v0.10.2

python-v0.10.3

python-v0.11.0

python-v0.11.1

python-v0.11.2

python-v0.11.3

python-v0.11.4

python-v0.11.5

python-v0.11.6

python-v0.12.0

python-v0.12.1

python-v0.13.0

python-v0.13.1

python-v0.13.2

python-v0.13.3

python-v0.13.3rc1

python-v0.2.0

python-v0.2.1

python-v0.3.0

python-v0.4.0

python-v0.4.1

python-v0.4.2

python-v0.5.0

python-v0.5.1

python-v0.5.2

python-v0.6.0

python-v0.7.0

python-v0.7.0-rc1

python-v0.7.0-rc2

python-v0.7.0-rc3

python-v0.7.0-rc4

python-v0.7.0-rc5

python-v0.7.0-rc6

python-v0.7.0-rc7

python-v0.8.0

python-v0.8.0-dev

python-v0.8.0-dev1

python-v0.8.0-dev2

python-v0.8.0-rc1

python-v0.8.0-rc2

python-v0.8.0-rc3

python-v0.8.0-transformers

python-v0.8.1

python-v0.8.1.rc1

python-v0.8.1.rc2

python-v0.9.0

python-v0.9.0.dev0

python-v0.9.0.dev1

python-v0.9.0.dev2

python-v0.9.0.dev3

python-v0.9.0.dev4

python-v0.9.0.rc1

python-v0.9.0.rc2

python-v0.9.1

python-v0.9.2

python-v0.9.3

python-v0.9.4

rust-v0.10.0

rust-v0.10.1

rust-v0.8.0

rust-v0.9.0

v0.0.10

v0.0.11

v0.0.12

v0.0.13

v0.0.3

v0.0.4

v0.0.5

v0.0.6

v0.0.7

v0.0.8

v0.0.9

v0.1.0

v0.1.1

v0.11.1

v0.11.2

v0.12.0

v0.13.0

v0.13.1

v0.13.2

v0.13.3

v0.13.4-rc1

v0.13.4-rc2

v0.13.4.rc1

v0.13.4.rc2

v0.13.4.rc3

v0.14.0

v0.14.0.rc1

v0.14.1

v0.14.1rc1

v0.15.0

v0.15.0.rc1

v0.15.1

v0.15.1.rc0

v0.15.1.rc1

v0.15.1rc0

v0.15.2

v0.15.2rc1

v0.16.0rc1

v0.19.0

v0.19.0rc0

v0.19.1

v0.19.1rc0

v0.20.0

v0.20.0rc1

v0.20.1

v0.20.1rc1

v0.20.2

v0.20.2rc1

v0.20.3

v0.20.3rc1

v0.20.4

v0.20.4-rc0

v0.20.4rc0

v0.21.0

v0.21.0rc0

v0.21.1

v0.21.1rc0

v0.21.2

v0.21.2rc0

v0.21.3

v0.21.4

v0.22.0

v0.22.0rc0

v0.22.1

v0.22.1rc0

v0.22.2

v0.22.2rc0

Mono Color

• 0ff2ab0f64 Fixing the stream by removing the read_index altogether. (#1716) Nicolas Patry 2025-01-09 17:41:15 +01:00
• 862d1a346a Fix panic in DecodeStream::step due to incorrect index usage (#1699) Sungyoon Jeong 2025-01-09 21:24:04 +09:00
• c04b97aab1 Update documentation of Rust feature (#1711) sondalex 2025-01-09 12:08:45 +01:00
• bdfc38b78d Fix typos (#1715) tinyboxvk 2025-01-09 06:53:20 -04:00
• 6945933829 update Split pretokenizer docstrings (#1701) Dylan-Harden3 2025-01-08 05:35:52 -06:00
• 166edd87c8 Fixing the README. (#1714) Nicolas Patry 2025-01-08 12:31:17 +01:00
• 3a6504d274 Upgrade to PyO3 0.23 (#1708) Nicolas Patry 2024-12-31 18:36:01 +01:00
• 555d44c47a Add feature flag hint to README.md, fixes #1633 (#1709) sftse 2024-12-30 17:01:53 +01:00
• 24d29f498d Update dev version and pyproject.toml (#1693) Arthur 2024-11-27 16:01:48 +01:00
• cf102e6725 proper release v0.21.0 git_v0.21.0 Arthur Zucker 2024-11-27 14:05:58 +01:00
• 2c750ab83c v0.21.0 Arthur Zucker 2024-11-27 13:54:49 +01:00
• 90795fe474 version = "0.21.0-rc0" v0.21.0rc0 Arthur Zucker 2024-11-27 13:42:29 +01:00
• 15e85b199c v0.21.0 Arthur Zucker 2024-11-27 13:37:33 +01:00
• 1bf2a66b80 v0.20.4-dev0 Arthur Zucker 2024-11-27 10:07:49 +01:00
• 350e5db82d v0.20.4 v0.20.4 git_v0.20.4-rc0 Arthur Zucker 2024-11-26 18:00:50 +01:00
• 8f026799eb update release workflow v0.20.4rc0 Arthur Zucker 2024-11-26 17:44:13 +01:00
• 6647c2006b v0.20.4rc0 Arthur Zucker 2024-11-26 17:27:48 +01:00
• 4bc7b4cc2a Fix encode_batch and encode_batch_fast to accept ndarrays again (#1679) Dimitris Iliopoulos 2024-11-21 05:55:11 -05:00
• eb4cc86d4e Bump cross-spawn from 6.0.5 to 6.0.6 in /bindings/node (#1687) dependabot[bot] 2024-11-25 10:04:06 +01:00
• ac34660e44 Fix encode_batch and encode_batch_fast to accept ndarrays again (#1679) Dimitris Iliopoulos 2024-11-21 05:55:11 -05:00
• f0c48bd89a Update README.md with install from source Arthur 2024-11-15 21:51:39 +01:00
• a1c572e2be 0.20.3-rc0 v0.20.4-rc0 git_v0.20.3-rc0 Nicolas Patry 2024-11-15 18:09:03 +07:00
• cc5fb01a2f Decode stream python (#1678) Nicolas Patry 2024-11-15 19:06:22 +08:00
• 500db282a8 Adding an API for decode streaming. (#1677) Nicolas Patry 2024-11-15 13:02:38 +08:00
• f4c9fd7f40 Testing ABI3 wheels to reduce number of wheels (#1674) Nicolas Patry 2024-11-15 13:02:22 +08:00
• 5aa9f6cff0 Disable caching for long strings. (#1676) Nicolas Patry 2024-11-07 21:36:27 +08:00
• c6b5c3eab7 More cache options. (#1675) Nicolas Patry 2024-11-06 18:12:09 +08:00
• 1740bff7a6 Revert "Upgrade python versions." Nicolas Patry 2024-11-06 13:18:03 +08:00
• b81ec467a6 Upgrade python versions. Nicolas Patry 2024-11-06 13:17:22 +08:00
• b63262a481 update cargo plus v0.20.3 v0.20.3 v0.20.3-release Arthur Zucker 2024-11-05 18:17:11 +01:00
• 5e97b53f0b update tag Arthur Zucker 2024-11-05 18:16:18 +01:00
• 6af367b0db cargo lock v0.20.3rc1 Arthur Zucker 2024-11-05 17:54:50 +01:00
• da6c367170 v0.20.3 Arthur Zucker 2024-11-05 16:26:11 +01:00
• 57884ebaa2 [MINOR:TYPO] Fix docstrings (#1653) Christopher Akiki 2024-11-05 16:25:06 +01:00
• 5e223ceb48 fix pylist (#1673) Arthur 2024-11-05 16:24:23 +01:00
• 0f3a3f957e update workflow Arthur Zucker 2024-11-04 18:38:32 +01:00
• 7c36735389 v0.20.2-dev.0 version Arthur Zucker 2024-11-04 18:36:40 +01:00
• caa650512c v 0.20.2 v0.20.2 python-313-release Arthur Zucker 2024-11-04 18:15:59 +01:00
• 5266b42a3d update python version install of setup python v0.20.2rc1 Arthur Zucker 2024-11-04 15:49:33 +01:00
• 4f7ba38119 some windows cna't support 3.13 :( Arthur Zucker 2024-11-04 15:14:40 +01:00
• 645487a0e0 cargo lock Arthur Zucker 2024-11-04 14:56:18 +01:00
• e7799d696e oups should be a RC1 first Arthur Zucker 2024-11-04 14:48:03 +01:00
• d2754da4e3 v0.20.2 + cargo Arthur Zucker 2024-11-04 14:34:44 +01:00
• f6f59ac51b update python release Arthur Zucker 2024-11-02 13:06:08 +01:00
• 6c15458868 Bump actions versions (#1669) tinyboxvk 2024-11-01 06:19:35 -03:00
• 6ade8c2d21 PyO3 0.22 (#1665) Dimitris Iliopoulos 2024-11-01 05:17:23 -04:00
• 41e0eaa561 Bump actions/checkout to v4 (#1667) tinyboxvk 2024-10-29 10:32:07 -03:00
• 5512a424bf Add safety comments (#1651) Manish Goregaokar 2024-10-29 01:44:06 -07:00
• 6ea758872d Unsound call of set_var (#1664) sftse 2024-10-25 15:44:30 +02:00
• a8738a95d1 Arg name correction: auth_token -> token (#1621) rravenel 2024-10-24 07:32:09 -07:00
• a56f73eded good defaults? fix-cache-issues Arthur Zucker 2024-10-21 15:45:23 +02:00
• b2c667cfbc update Arthur Zucker 2024-10-21 15:18:43 +02:00
• c81c34a233 for now compile but breaking Arthur Zucker 2024-10-21 15:05:26 +02:00
• 714a3bd0c3 use sysinfo to pre-allocate a big enough cache Arthur Zucker 2024-10-21 14:53:52 +02:00
• ab3236f640 propagate to unigram Arthur Zucker 2024-10-21 11:57:29 +02:00
• 6e0175acc5 add a clear_cache function! Arthur Zucker 2024-10-21 11:47:36 +02:00
• 9b77c054ef Fix off-by-one error in tokenizer::normalizer::Range::len (#1638) Ryan Landay 2024-10-14 02:40:17 -04:00
• bce68a60cb Bump cookie and express in /tokenizers/examples/unstable_wasm/www (#1648) dependabot[bot] 2024-10-10 15:30:24 +02:00
• 51826532d4 push new dev version Arthur Zucker 2024-10-10 12:00:16 +02:00
• d98298a2c2 0.20.1 v0.20.1 branch_0.20.1.rc1 Arthur Zucker 2024-10-10 11:45:24 +02:00
• de305f2170 update to ubuntu-22.04 v0.20.1rc1 Arthur Zucker 2024-10-10 11:27:06 +02:00
• 1053470ff7 use --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }} Arthur Zucker 2024-10-10 11:17:09 +02:00
• f7c33eb3b2 add Cargo Arthur Zucker 2024-10-10 10:14:44 +02:00
• eca17be37b v 0.20.1-rc1 Arthur Zucker 2024-10-10 10:14:01 +02:00
• 81d83361d0 fix the unigram::from calls assign-token Arthur Zucker 2024-10-05 17:58:22 +02:00
• 167ecdebfb small fixed Arthur Zucker 2024-10-05 17:56:06 +02:00
• 0475c057dd fix added vocab tests Arthur Zucker 2024-10-05 17:17:52 +02:00
• e8933fa5b9 potential initial solution for the annoying unigram model :) Arthur Zucker 2024-10-05 17:16:31 +02:00
• ee7ce80e0b forgot to remove from added tokens map! Arthur Zucker 2024-10-04 15:55:43 +02:00
• 545d7230f4 fix unwrap errors Arthur Zucker 2024-10-04 15:24:14 +02:00
• ed34ffd334 add a small test Arthur Zucker 2024-10-04 15:00:35 +02:00
• b5640a65cf simplify the logic Arthur Zucker 2024-10-04 14:46:42 +02:00
• 6d48e58219 remove print Arthur Zucker 2024-07-12 10:47:38 +02:00
• 2d4b3735e4 fix everything Arthur Zucker 2024-07-12 10:38:40 +02:00
• 4190db7ddd pass compilation Arthur Zucker 2024-07-12 10:12:29 +02:00
• 4794ed516f fix Arthur Zucker 2024-07-12 10:07:27 +02:00
• b359bde47a nit Arthur Zucker 2024-07-12 09:58:06 +02:00
• ddab901338 current update Arthur Zucker 2024-07-12 09:49:42 +02:00
• 97e8818ecf add python bindongs as well Arthur Zucker 2024-07-12 08:52:04 +02:00
• fc0f0656f0 allow to assign a new token Arthur Zucker 2024-07-12 08:08:14 +02:00
• 557fde76d8 style: simplify string formatting for readability (#1632) Hamir Mahal 2024-10-04 04:11:50 -07:00
• 3d51a1695f Fix documentation build (#1642) Arthur 2024-10-01 14:48:02 +02:00
• 294ab86fe0 Bump webpack in /tokenizers/examples/unstable_wasm/www (#1641) dependabot[bot] 2024-10-01 14:17:23 +02:00
• 2b97a5e49e Bump send and express in /tokenizers/examples/unstable_wasm/www (#1631) dependabot[bot] 2024-10-01 14:17:09 +02:00
• 077678d1d1 Bump serve-static and express in /tokenizers/examples/unstable_wasm/www (#1630) dependabot[bot] 2024-10-01 14:16:53 +02:00
• 2204066e78 Bump body-parser and express in /tokenizers/examples/unstable_wasm/www (#1629) dependabot[bot] 2024-10-01 14:16:41 +02:00
• 3fb1371c1c [ignore_merges] Fix offsets (#1640) Arthur 2024-10-01 09:22:20 +02:00
• b4a38c4f63 Bump actions/download-artifact from 3 to 4.1.7 in /.github/workflows (#1626) dependabot[bot] 2024-09-30 16:38:28 +02:00
• ac4e717070 potential solution? fix-split-special Arthur Zucker 2024-09-30 14:31:47 +02:00
• 14a07b06e4 fix filelink (#1610) 152334H 2024-08-12 05:35:33 +00:00
• 75aef5b75b Update README.md (#1608) Arthur 2024-08-09 10:40:21 +02:00
• a5adaace3d version 0.20.0 v0.20.0 branch_v0.20.0.rc1 Arthur Zucker 2024-08-08 18:42:29 +02:00
• 81c471cf17 update dev version 0.20.0 Arthur Zucker 2024-08-08 18:10:55 +02:00
• 85cc05a32f Fix CI (#1607) Nicolas Patry 2024-08-08 17:09:30 +02:00
• a8def0739f Merge branch 'fix_release' of github.com:huggingface/tokenizers into branch_v0.20.0.rc1 v0.20.0rc1 Arthur Zucker 2024-08-08 16:57:51 +02:00
• fe5067347e Fix CI Nicolas Patry 2024-08-08 16:55:47 +02:00
• b253835968 push cargo Arthur Zucker 2024-08-08 16:34:33 +02:00
• fc3bb7653c update dependencies Arthur Zucker 2024-08-08 16:33:01 +02:00
• bfd9cdeefb Perf improvement 16% by removing offsets. (#1587) Nicolas Patry 2024-08-08 14:56:13 +02:00
• b6d01b788a initial commit fast-regex Arthur Zucker 2024-06-20 16:01:51 +02:00