add first 500 tokens
Browse files
app.py
CHANGED
|
@@ -367,6 +367,10 @@ def main():
|
|
| 367 |
first_10_tokens = input_ids[:10]
|
| 368 |
first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
|
| 369 |
st.write(first_10_tokens_text)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 370 |
st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
|
| 371 |
st.write(
|
| 372 |
" chunk_size=%s"
|
|
|
|
| 367 |
first_10_tokens = input_ids[:10]
|
| 368 |
first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
|
| 369 |
st.write(first_10_tokens_text)
|
| 370 |
+
st.write("First 500 tokens:")
|
| 371 |
+
first_500_tokens = input_ids[:500]
|
| 372 |
+
first_500_tokens_text = tokenizer.convert_ids_to_tokens(first_500_tokens)
|
| 373 |
+
st.write(first_500_tokens_text)
|
| 374 |
st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
|
| 375 |
st.write(
|
| 376 |
" chunk_size=%s"
|