I am trying to enter a series of Topics into ChromaDB, those topics can be found here There are 35 total topics, 34 of which are unique. The topics are in valid JSON format – I can add and remove them from MongoDB, use JSON dumps and loads on them, and they were created from a langchain openai call where the output comes from a JSON formatter.

No matter what I try, however, only 7 of them are entered into ChromaDB AND trying to use a RecursiveJsonSplitter always results in an error.

Here are the two methods I am using (the write_object_to_prompt call simply removes all curly brackets and add in line breaks/indents):

def add_to_chroma(database_name: str, collection_name: str, json_object: json, user_directory: str, state, meta_data: json, doc_ids: list[str] = None):
    try:
        db_directory = os.path.join(user_directory, database_name + ".db")
        embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
        # embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
        chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
                           collection_metadata={"hnsw:space": "cosine"})
        embed_object = write_object_to_prompt(json_object)
        text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
        docs = text_splitter.create_documents(texts=[embed_object], metadatas=[meta_data])

        if doc_ids is None:
            doc_ids = [str(uuid.uuid4()) for i in range(1, len(docs) + 1)]
        else:
            # We look to see if the document exists:
            result = chroma_db.get(doc_ids)
            if result is not None and len(result) > 0:
                # This is an update:
                state["persistent_logs"].append("Updating " + meta_data["topic_id"] + " in Chroma")
                chroma_db.update_documents(doc_ids, docs)
                return doc_ids
        state["persistent_logs"].append("Adding " + meta_data["topic_id"] + " to Chroma")
        chroma_db.from_documents(docs, embedding_function, ids=doc_ids)
    except:
        trace_back = traceback.format_exc()
        logging.error("An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
                      "nHere is the document that failed: " + write_object_to_prompt(json_object) + " nWith the error:n " + trace_back)
        state["persistent_logs"].append(
            "An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
            "nHere is the document that failed: " + write_object_to_prompt(json_object) + " nWith the error:n " + trace_back)
    return doc_ids, state

def get_current_count(database_name: str, collection_name: str, user_directory: str) -> int:
    db_directory = os.path.join(user_directory, database_name + ".db")
    # embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
    embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
    # Cosine will keep te similarity scores between zero and one
    chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
                       collection_metadata={"hnsw:space": "cosine"})
    results = chroma_db.get()
    total_count = 0
    if results is not None:
        total_count = len(results)
    return total_count

And here is my testing script:

user_directory = "../UserData/user-x"
with open("Topics.txt", "r") as f:
    Topics = json.load(f)
print("Total topics: " + str(len(Topics)))
state = {
        "errors": "",
        "persistent_logs": [],
    }
unique_ids = []
for this_topic in Topics:
    meta = {"topic_id": this_topic["topic_id"]}
    doc_ids, state = add_to_chroma("user-x", "conversations", this_topic, user_directory, state, meta)
    if this_topic["topic_id"] not in unique_ids:
        unique_ids.append(this_topic["topic_id"])
print("Number of unique Ids: " + str(len(unique_ids)))

if len(state["persistent_logs"]) > 0:
    for log in state["persistent_logs"]:
        print(log)
print("Total Topics in Chroma " + str(get_current_count("user-x", "topics", user_directory)))

I never get any errors back from Chroma and the output shows 35 entries.

I have tried to use OpenAi embeddings, HuggingFace embeddings, JSON and text splitters, creating ChomraDB Documents manually. Entering the JSON as plaintext, entering it as JSON. Nothing is working.

Does anybody have any ideas?