` this is embedding.ts

import { OpenAIApi, Configuration } from "openai-edge";


const config = new Configuration({
  apiKey: process.env.OPENAI_API_KEY,
});

const openai = new OpenAIApi(config);

export async function getEmbeddings(text: string) {
  try {
    const response = await openai.createEmbedding({
      model: "text-embedding-ada-002",
      input: text.replace(/n/g, " "),
    });
    const result = await response.json();
    return result.data[0].embedding as number[];
  } catch (error) {
    console.log("error calling openai embeddings api", error);
    throw error;
  }
}

this is pinecone.ts:-

  import { Pinecone, PineconeRecord } from "@pinecone-database/pinecone";
import { downloadFromS3 } from "./s3-server";
import { PDFLoader } from "langchain/document_loaders/fs/pdf";
import md5 from "md5";
import {
  Document,
  RecursiveCharacterTextSplitter,
} from "@pinecone-database/doc-splitter";
import { getEmbeddings } from "./embeddings";
import { convertToAscii } from "./utils";

export const getPineconeClient = () => {
  return new Pinecone({
    environment: process.env.PINECONE_ENVIRONMENT!,
    apiKey: process.env.PINECONE_API_KEY!,
  });
};

type PDFPage = {
  pageContent: string;
  metadata: {
    loc: { pageNumber: number };
  };
};

export async function loadS3IntoPinecone(fileKey: string) {
  // 1. obtain the pdf -> downlaod and read from pdf
  console.log("downloading s3 into file system");
  const file_name = await downloadFromS3(fileKey);
  if (!file_name) {
    throw new Error("could not download from s3");
  }
  console.log("loading pdf into memory" + file_name);
  const loader = new PDFLoader(file_name);
  const pages = (await loader.load()) as PDFPage[];

  // 2. split and segment the pdf
  const documents = await Promise.all(pages.map(prepareDocument));

  // 3. vectorise and embed individual documents
  const vectors = await Promise.all(documents.flat().map(embedDocument));

  // 4. upload to pinecone
  const client = await getPineconeClient();
  const pineconeIndex = await client.index("chattpdf");
  const namespace = pineconeIndex.namespace(convertToAscii(fileKey));

  console.log("inserting vectors into pinecone");
  await namespace.upsert(vectors);

  return documents[0];
}

async function embedDocument(doc: Document) {
  try {
    const embeddings = await getEmbeddings(doc.pageContent);
    const hash = md5(doc.pageContent);

    return {
      id: hash,
      values: embeddings,
      metadata: {
        text: doc.metadata.text,
        pageNumber: doc.metadata.pageNumber,
      },
    } as PineconeRecord;
  } catch (error) {
    console.log("error embedding document", error);
    throw error;
  }
}

export const truncateStringByBytes = (str: string, bytes: number) => {
  const enc = new TextEncoder();
  return new TextDecoder("utf-8").decode(enc.encode(str).slice(0, bytes));
};

async function prepareDocument(page: PDFPage) {
  let { pageContent, metadata } = page;
  pageContent = pageContent.replace(/n/g, "");
  // split the docs
  const splitter = new RecursiveCharacterTextSplitter();
  const docs = await splitter.splitDocuments([
    new Document({
      pageContent,
      metadata: {
        pageNumber: metadata.loc.pageNumber,
        text: truncateStringByBytes(pageContent, 36000),
      },
    }),
  ]);
  return docs;
}

this is route.ts

import { db } from "@/lib/db";
import { chats } from "@/lib/db/schema";
import { loadS3IntoPinecone } from "@/lib/pinecone";
import { getS3Url } from "@/lib/s3";
import { auth } from "@clerk/nextjs";
import { NextResponse } from "next/server";

// /api/create-chat
export async function POST(req: Request, res: Response) {
  const { userId } = await auth();
  if (!userId) {
    return NextResponse.json({ error: "unauthorized" }, { status: 401 });
  }
  try {
    const body = await req.json();
    const { file_key, file_name } = body;
    console.log(file_key, file_name);
    await loadS3IntoPinecone(file_key);
    const chat_id = await db
      .insert(chats)
      .values({
        fileKey: file_key,
        pdfName: file_name,
        pdfUrl: getS3Url(file_key),
        userId,
      })
      .returning({
        insertedId: chats.id,
      });

    return NextResponse.json(
      {
        chat_id: chat_id[0].insertedId,
      },
      { status: 200 }
    );
  } catch (error) {
    console.error(error);
    return NextResponse.json(
      { error: "internal server error" },
      { status: 500 }
    );
  }
}

this is the error:-

• TypeError: Cannot read properties of undefined (reading ‘0’)
  at getEmbeddings (webpack-internal:///(rsc)/./src/lib/embeddings.ts:18:27)
  at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
  at async embedDocument (webpack-internal:///(rsc)/./src/lib/pinecone.ts:54:28)
  at async Promise.all (index 7)
  at async loadS3IntoPinecone (webpack-internal:///(rsc)/./src/lib/pinecone.ts:43:21)
  at async POST (webpack-internal:///(rsc)/./src/app/api/create-chat/route.ts:31:9)
  at async C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistcompilednext-serverapp-route.runtime.dev.js:6:53446
  at async e_.execute (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistcompilednext-serverapp-route.runtime.dev.js:6:44747)
  at async e_.handle (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistcompilednext-serverapp-route.runtime.dev.js:6:54700)
  at async doRender (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:1377:42)
  at async cacheEntry.responseCache.get.routeKind (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:1599:28)
  at async DevServer.renderToResponseWithComponentsImpl (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:1507:28)
  at async DevServer.renderPageComponent (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:1924:24)
  at async DevServer.renderToResponseImpl (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:1962:32)
  at async DevServer.pipeImpl (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:920:25)
  at async NextNodeServer.handleCatchallRenderRequest (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistservernext-server.js:272:17)
  at async DevServer.handleRequestImpl (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverbase-server.js:816:17)
  at async C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverdevnext-dev-server.js:339:20
  at async Span.traceAsyncFn (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdisttracetrace.js:154:20)
  at async DevServer.handleRequest (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverdevnext-dev-server.js:336:24)
  at async invokeRender (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverlibrouter-server.js:174:21)
  at async handleRequest (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverlibrouter-server.js:353:24)
  at async requestHandlerImpl (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverlibrouter-server.js:377:13)
  at async Server.requestListener (C:UsershpDownloadsnewmanchattpdfnode_modulesnextdistserverlibstart-server.js:141:13)
  POST /api/create-chat 500 in 4005ms

`