I’m developing a Python API using FastAPI to read a CSV file containing a list of medicines approved by the Brazilian government. The CSV file is in Portuguese, which includes special characters such as á, é, ã, etc. However, I’m encountering an issue where Python doesn’t handle these characters correctly by default. This leads to incorrect string comparisons between hardcoded strings and those read from the CSV file.

For instance, the CSV file contains the word “GENÉRICO,” but when read, it appears as “GENÃRICO.” Consequently, string comparisons fail even though the strings should match.

from pydantic import BaseModel
from typing import List, Optional, Annotated
import pandas as pd

app = FastAPI()

# Load CSV data with the correct delimiter and encoding
file_path = 'data/DADOS_ABERTOS_MEDICAMENTOS_FILTRADO.csv'
df = pd.read_csv(file_path, encoding='latin1', sep=',')

# Normalize the 'CATEGORIA_REGULATORIA' column to avoid matching issues
df['CATEGORIA_REGULATORIA'] = df['CATEGORIA_REGULATORIA'].str.upper().str.strip()

# Debug: Display unique regulatory categories
categorias_unicas = df['CATEGORIA_REGULATORIA'].unique()
print("Unique Regulatory Categories:")
for categoria in categorias_unicas:
    print(categoria)

class Medicamento(BaseModel):
    novo: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Novo",
            description="List of new medicines returned",
        ),
    ] = None
    similar: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Similar",
            description="List of similar medicines returned",
        ),
    ] = None
    fitoterapico: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Fitoterápico",
            description="List of herbal medicines returned",
        ),
    ] = None
    especifico: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Específico",
            description="List of specific medicines returned",
        ),
    ] = None
    generico: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Genérico",
            description="List of generic medicines returned",
        ),
    ] = None
    biologico: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Biológico",
            description="List of biological medicines returned",
        ),
    ] = None
    dinamizado: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Dinamizado",
            description="List of dynamized medicines returned",
        ),
    ] = None
    radiofarmaco: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Radiofármaco",
            description="List of radiopharmaceutical medicines returned",
        ),
    ] = None
    produto_de_t: Annotated[
        Optional[List[str]],
        Query(
            alias="item-query",
            title="Produto de T",
            description="List of T products returned",
        ),
    ] = None

@app.get("/medicamentos/", response_model=Medicamento)
async def get_medicamentos(principio_ativo: str):
    # Check if 'PRINCIPIO_ATIVO' column is correctly read
    if 'PRINCIPIO_ATIVO' not in df.columns:
        return {"error": "Column 'PRINCIPIO_ATIVO' not found in DataFrame"}

    # Filter DataFrame based on active ingredient
    df_filtered = df[df['PRINCIPIO_ATIVO'].str.contains(principio_ativo, case=False, na=False)]

    # Debug: Print filtered rows
    print(f"Filtered rows for active ingredient '{principio_ativo}':")
    print(df_filtered)

    # Debug: Print categories in filtered rows
    categorias_filtradas = df_filtered['CATEGORIA_REGULATORIA'].unique()
    print("Categories in filtered rows:")
    for categoria in categorias_filtradas:
        print(categoria)

    # Create lists of medicines for each regulatory category
    novo = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'NOVO']['NOME_PRODUTO'].tolist()
    similar = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'SIMILAR']['NOME_PRODUTO'].tolist()
    fitoterapico = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'FITOTERÁPICO']['NOME_PRODUTO'].tolist()
    especifico = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'ESPECÍFICO']['NOME_PRODUTO'].tolist()
    generico = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'GENÉRICO']['NOME_PRODUTO'].tolist()
    biologico = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'BIOLÓGICO']['NOME_PRODUTO'].tolist()
    dinamizado = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'DINAMIZADO']['NOME_PRODUTO'].tolist()
    radiofarmaco = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'RADIOFÁRMACO']['NOME_PRODUTO'].tolist()
    produto_de_t = df_filtered[df_filtered['CATEGORIA_REGULATORIA'] == 'PRODUTO DE T']['NOME_PRODUTO'].tolist()

    # Return lists of medicines
    return Medicamento(
        novo=novo if novo else None,
        similar=similar if similar else None,
        fitoterapico=fitoterapico if fitoterapico else None,
        especifico=especifico if especifico else None,
        generico=generico if generico else None,
        biologico=biologico if biologico else None,
        dinamizado=dinamizado if dinamizado else None,
        radiofarmaco=radiofarmaco if radiofarmaco else None,
        produto_de_t=produto_de_t if produto_de_t else None
    )

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

I attempted to look for a PRINCIPIO_ATIVO called “DIPIRONA,” and I’m getting the following response:

{
  "novo": [
    "APRACUR",
    "BENEGRIP",
    "BUSCOPAN COMPOSTO",
    "CAFILISADOR",
    "CEFALIV",
    "DORALGINA DIPCAF",
    "DORFLEX",
    "DORIL DC 500",
    "MIONEVRIX",
    "NOVALGINA",
    "MIRADOR CÃLICA",
    "DORFLEX"
  ],
  "similar": [
    "ABERALGINA",
    "ALGEXIN COMPOSTO",
    "ALIVDIP",
    "ALIVDIP",
    "ANA - FLEX",
    "ANADOR",
    "APRACUR DUO",
    "ATROVERAN DIP",
    "ATROVERAN DIP",
    "ATROVEX",
    "BELSPAN",
    "BINOSPAN COMPOSTO",
    "BUSCOPLEX COMPOSTO",
    "BUSCOVERAN COMPOSTO",
    "BUSCOVERAN COMPOSTO",
    "CONMEL",
    "DEXALGEN",
    "DIFEBRIL",
    "DIPIDOR",
    "DIPIFARMA",
    "DIPIMED",
    "DIPIRALGIN",
    "DIPIRONATI",
    "DIPRIN",
    "DORALEX",
    "DORALGINA",
    "DORFEBRIL",
    "DORFLEX UNO",
    "DORICIN",
    "DORICIN UNO",
    "DORILEN",
    "DORILEN DIP",
    "DORILESS",
    "DORONA CAFI",
    "DORSPAN",
    "DORTRIRELAX",
    "DRENOGRIP",
    "ENXAK",
    "ESCOPEN COMPOSTO",
    "FENAFLEX - ODC",
    "FURP- DIPIRONA",
    "GRIPINEW",
    "HIOARISTON",
    "HIOSPAN COMPOSTO",
    "HYNALGIN",
    "HYPOCINA COMPOSTA",
    "LISADOR",
    "LISADOR DIP",
    "LISADOR DIP",
    "LQFEX - DIPIRONA",
    "MAGNOPYROL",
    "MAXALGINA",
    "MEDALGÃSICO",
    "MEDALGÃSICO",
    "MEDALGÃSICO",
    "MIGRALIV",
    "MIORRELAX",
    "MIORRELAX UNO",
    "MIRADOR",
    "NEOCOPAN COMPOSTO",
    "NEOCOPAN COMPOSTO",
    "NEOSALDINA",
    "NEOSALDINA DIP",
    "NERALGYN",
    "NEVRALGEX",
    "NOFEBRIN",
    "NOGRIPE",
    "NOVRALFLEX",
    "RELAFLEX",
    "RELAXMED",
    "SANTIDOR",
    "SEDADOR",
    "SEDALEX",
    "SEDOL",
    "SUALIV",
    "Sedamed",
    "TERMOPIRONA",
    "TROPINAL",
    "LISADOR CÃLICA",
    "DORSPAN COMPOSTO",
    "COLIHELP",
    "DODOY",
    "DORENXAQ",
    "MULTIPIRAL",
    "RESSALIVDOR",
    "DORFLEX DIP",
    "ESPASMOPAN COMPOSTO",
    "ESPASMOPAN COMPOSTO",
    "NEVRALGEX DIP",
    "DORAGEO",
    "DORALGINA",
    "IZENXAQ",
    "DORCIFLEXIN",
    "NEVRALGEX DC",
    "XAQUELIV",
    "ANADOR",
    "DORFLEX DIP",
    "DORFLEX UNO"
  ],
  "fitoterapico": null,
  "especifico": null,
  "generico": null,
  "biologico": null,
  "dinamizado": null,
  "radiofarmaco": null,
  "produto_de_t": null
}

As you can see, the generico list is not being populated even though there should be a list of generic medicines.