├── app ├── __init__.py ├── schemas.py ├── database.py ├── models.py ├── crud.py ├── main.py └── utils.py ├── tests ├── __init__.py └── test_app.py ├── alembic ├── README ├── script.py.mako ├── versions │ ├── 11fd56360959_initial_migration.py │ ├── edd09b39f411_initial_migration.py │ ├── cfc4d31835bc_initial_migration.py │ ├── 252ea3c4cd46_initial_migration.py │ ├── 48db5508e2d2_initial_migration.py │ ├── 12dc89c2a759_initial_migration.py │ ├── 4bcc7e133b31_initial_migration.py │ ├── 5f3ea3ef5ae7_initial_migration.py │ ├── acfb245c8e0f_initial_migration.py │ ├── ee309898ab0f_initial_migration.py │ └── 7792e74809ac_initial_migration.py └── env.py ├── requirements.txt ├── Dockerfile ├── docker-compose.yml ├── .gitignore ├── README.md └── alembic.ini /app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /alembic/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn[standard] 3 | sqlalchemy 4 | psycopg2-binary 5 | alembic 6 | python-dotenv 7 | asyncpg 8 | sqlmodel 9 | requests 10 | beautifulsoup4 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-bullseye 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt requirements.txt 6 | RUN pip install --progress-bar off --no-cache-dir -r requirements.txt 7 | 8 | COPY . . 9 | 10 | CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] -------------------------------------------------------------------------------- /app/schemas.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional 3 | 4 | class ScrapeRequest(BaseModel): 5 | url: str 6 | 7 | class SongCreate(BaseModel): 8 | name: str 9 | artist: str 10 | year: Optional[int] = None 11 | 12 | class EmailRequest(BaseModel): 13 | email: str 14 | subject: str 15 | message: str -------------------------------------------------------------------------------- /app/database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | from sqlalchemy import create_engine 5 | from dotenv import load_dotenv 6 | import os 7 | 8 | load_dotenv() 9 | 10 | DB_URL = os.getenv("DB_URL") 11 | 12 | engine = create_async_engine(DB_URL, echo=True) 13 | 14 | async_session = sessionmaker( 15 | bind=engine, 16 | class_=AsyncSession, 17 | expire_on_commit=False, 18 | ) 19 | Base = declarative_base() -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | web: 5 | build: . 6 | command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload 7 | volumes: 8 | - .:/app 9 | ports: 10 | - "8000:8000" 11 | env_file: 12 | - .env 13 | depends_on: 14 | - db 15 | 16 | db: 17 | image: postgres:15-bullseye 18 | volumes: 19 | - postgres_data:/var/lib/postgresql/data 20 | environment: 21 | POSTGRES_USER: ${POSTGRES_USER} 22 | POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} 23 | POSTGRES_DB: ${POSTGRES_DB} 24 | ports: 25 | - "5432:5432" 26 | 27 | volumes: 28 | postgres_data: -------------------------------------------------------------------------------- /app/models.py: -------------------------------------------------------------------------------- 1 | from sqlmodel import SQLModel, Field 2 | from sqlalchemy import Column, JSON 3 | from typing import List, Optional 4 | 5 | class SongBase(SQLModel): 6 | name: str 7 | artist: str 8 | year: Optional[int] = None 9 | 10 | class Song(SongBase, table=True): 11 | id: int = Field(default=None, nullable=False, primary_key=True) 12 | 13 | class ScrapedData(SQLModel, table=True): 14 | id: Optional[int] = Field(default=None, primary_key=True) 15 | title: Optional[str] 16 | meta_description: Optional[str] 17 | headings: Optional[List[str]] = Field(sa_column=Column(JSON)) 18 | links: Optional[List[str]] = Field(sa_column=Column(JSON)) 19 | content: Optional[List[str]] = Field(sa_column=Column(JSON)) -------------------------------------------------------------------------------- /alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | ${imports if imports else ""} 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = ${repr(up_revision)} 16 | down_revision: Union[str, None] = ${repr(down_revision)} 17 | branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} 18 | depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} 19 | 20 | 21 | def upgrade() -> None: 22 | ${upgrades if upgrades else "pass"} 23 | 24 | 25 | def downgrade() -> None: 26 | ${downgrades if downgrades else "pass"} 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | *.py[cod] 3 | __pycache__/ 4 | *.so 5 | *.egg 6 | *.egg-info/ 7 | dist/ 8 | build/ 9 | .eggs/ 10 | *.pyc 11 | 12 | # Environment variables 13 | .env 14 | .env.* 15 | 16 | # Virtual environment 17 | venv/ 18 | ENV/ 19 | env/ 20 | .Python 21 | 22 | # VS Code 23 | .vscode/ 24 | 25 | # PyCharm 26 | .idea/ 27 | *.iml 28 | 29 | # Logs 30 | logs/ 31 | *.log 32 | 33 | # Database 34 | *.sqlite3 35 | 36 | # Alembic 37 | alembic/versions/*.pyc 38 | 39 | # Docker 40 | docker-compose.override.yml 41 | *.log 42 | 43 | # Pytest cache 44 | .pytest_cache/ 45 | 46 | # Jupyter Notebook 47 | .ipynb_checkpoints 48 | 49 | # macOS 50 | .DS_Store 51 | .AppleDouble 52 | .LSOverride 53 | 54 | # Windows 55 | Thumbs.db 56 | ehthumbs.db 57 | Desktop.ini 58 | $RECYCLE.BIN/ 59 | 60 | # Backup files 61 | *.bak 62 | *.swp 63 | *.swo 64 | *~ -------------------------------------------------------------------------------- /alembic/versions/11fd56360959_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 11fd56360959 4 | Revises: ee309898ab0f 5 | Create Date: 2024-08-17 17:38:06.971839 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = '11fd56360959' 16 | down_revision: Union[str, None] = 'ee309898ab0f' 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | pass 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | pass 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /alembic/versions/edd09b39f411_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: edd09b39f411 4 | Revises: 11fd56360959 5 | Create Date: 2024-08-18 16:38:48.097139 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = 'edd09b39f411' 16 | down_revision: Union[str, None] = '11fd56360959' 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | pass 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | pass 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /alembic/versions/cfc4d31835bc_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: cfc4d31835bc 4 | Revises: 5 | Create Date: 2024-08-16 02:15:41.935920 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = 'cfc4d31835bc' 16 | down_revision: Union[str, None] = None 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | op.create_table( 23 | 'song', 24 | sa.Column('id', sa.Integer(), primary_key=True, nullable=False), 25 | sa.Column('name', sa.String(), nullable=False), 26 | sa.Column('artist', sa.String(), nullable=False), 27 | sa.Column('year', sa.Integer(), nullable=True) 28 | ) 29 | 30 | def downgrade() -> None: 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.drop_table('song') 33 | # ### end Alembic commands ### 34 | -------------------------------------------------------------------------------- /alembic/versions/252ea3c4cd46_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 252ea3c4cd46 4 | Revises: cfc4d31835bc 5 | Create Date: 2024-08-16 04:46:03.292972 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = '252ea3c4cd46' 15 | down_revision: Union[str, None] = 'cfc4d31835bc' 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('scrapeddata', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('title', sa.String(), nullable=True), 24 | sa.Column('meta_description', sa.String(), nullable=True), 25 | sa.Column('headings', sa.String(), nullable=True), 26 | sa.Column('links', sa.String(), nullable=True), 27 | sa.Column('content', sa.String(), nullable=True), 28 | sa.PrimaryKeyConstraint('id') 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | def downgrade() -> None: 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_table('scrapeddata') 35 | # ### end Alembic commands ### -------------------------------------------------------------------------------- /alembic/versions/48db5508e2d2_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 48db5508e2d2 4 | Revises: 7792e74809ac 5 | Create Date: 2024-08-16 06:08:37.337340 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = '48db5508e2d2' 15 | down_revision: Union[str, None] = '7792e74809ac' 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table( 23 | 'scrapeddata', 24 | sa.Column('id', sa.Integer(), nullable=False), 25 | sa.Column('title', sa.String(), nullable=True), 26 | sa.Column('meta_description', sa.String(), nullable=True), 27 | sa.Column('headings', sa.Text(), nullable=True), 28 | sa.Column('links', sa.Text(), nullable=True), 29 | sa.Column('content', sa.Text(), nullable=True), 30 | sa.PrimaryKeyConstraint('id') 31 | ) 32 | # ### end Alembic commands ### 33 | 34 | 35 | def downgrade() -> None: 36 | # ### commands auto generated by Alembic - please adjust! ### 37 | op.drop_table('scrapeddata') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /alembic/versions/12dc89c2a759_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 12dc89c2a759 4 | Revises: 4bcc7e133b31 5 | Create Date: 2024-08-16 07:02:03.149004 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | import sqlmodel 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = '12dc89c2a759' 17 | down_revision: Union[str, None] = '4bcc7e133b31' 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.create_table('scrapeddata', 25 | sa.Column('id', sa.Integer(), nullable=False), 26 | sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 27 | sa.Column('meta_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 28 | sa.Column('headings', sa.Text(), nullable=True), 29 | sa.Column('links', sa.Text(), nullable=True), 30 | sa.Column('content', sa.Text(), nullable=True), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('scrapeddata') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /alembic/versions/4bcc7e133b31_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 4bcc7e133b31 4 | Revises: acfb245c8e0f 5 | Create Date: 2024-08-16 06:56:05.360043 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | import sqlmodel 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = '4bcc7e133b31' 17 | down_revision: Union[str, None] = 'acfb245c8e0f' 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.create_table('scrapeddata', 25 | sa.Column('id', sa.Integer(), nullable=False), 26 | sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 27 | sa.Column('meta_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 28 | sa.Column('headings', sa.JSON(), nullable=True), 29 | sa.Column('links', sa.JSON(), nullable=True), 30 | sa.Column('content', sa.JSON(), nullable=True), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('scrapeddata') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /alembic/versions/5f3ea3ef5ae7_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 5f3ea3ef5ae7 4 | Revises: 12dc89c2a759 5 | Create Date: 2024-08-16 14:53:38.233522 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | import sqlmodel 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = '5f3ea3ef5ae7' 17 | down_revision: Union[str, None] = '12dc89c2a759' 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.create_table('scrapeddata', 25 | sa.Column('id', sa.Integer(), nullable=False), 26 | sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 27 | sa.Column('meta_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 28 | sa.Column('headings', sa.JSON(), nullable=True), 29 | sa.Column('links', sa.JSON(), nullable=True), 30 | sa.Column('content', sa.JSON(), nullable=True), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('scrapeddata') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /alembic/versions/acfb245c8e0f_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: acfb245c8e0f 4 | Revises: 48db5508e2d2 5 | Create Date: 2024-08-16 06:46:47.695536 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | import sqlmodel 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = 'acfb245c8e0f' 17 | down_revision: Union[str, None] = '48db5508e2d2' 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.create_table('scrapeddata', 25 | sa.Column('id', sa.Integer(), nullable=False), 26 | sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 27 | sa.Column('meta_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 28 | sa.Column('headings', sa.Text(), nullable=True), 29 | sa.Column('links', sa.Text(), nullable=True), 30 | sa.Column('content', sa.Text(), nullable=True), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('scrapeddata') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /alembic/versions/ee309898ab0f_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: ee309898ab0f 4 | Revises: 5f3ea3ef5ae7 5 | Create Date: 2024-08-16 15:07:13.974613 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | import sqlmodel 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = 'ee309898ab0f' 17 | down_revision: Union[str, None] = '5f3ea3ef5ae7' 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.create_table('scrapeddata', 25 | sa.Column('id', sa.Integer(), nullable=False), 26 | sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 27 | sa.Column('meta_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True), 28 | sa.Column('headings', sa.JSON(), nullable=True), 29 | sa.Column('links', sa.JSON(), nullable=True), 30 | sa.Column('content', sa.JSON(), nullable=True), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('scrapeddata') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Experimento FastAPI 2 | 3 | Aplicação construída com FastAPI e SQLModel, utilizando Docker para facilitar o setup do ambiente. Ela permite fazer scraping de conteúdo da web e armazenar os dados em um banco de dados PostgreSQL. Também possibilita o envio de e-mails de forma assíncrona em segundo plano. 4 | 5 | ## Requisitos 6 | 7 | - Docker 8 | - Docker Compose 9 | 10 | ## Instalação 11 | 12 | ### Clonar o Repositório 13 | 14 | ``` 15 | git clone https://github.com/the-akira/FastAPI.git 16 | cd FastAPI 17 | ``` 18 | 19 | ### Configurar Variáveis de Ambiente 20 | 21 | Crie um arquivo `.env` na raiz do projeto com o seguinte conteúdo: 22 | 23 | ``` 24 | POSTGRES_USER=myuser 25 | POSTGRES_PASSWORD=mypassword 26 | POSTGRES_DB=mydatabase 27 | DB_URL=postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} 28 | EMAIL=enderecoemail 29 | EMAIL_PASSWORD=senhaemail 30 | ``` 31 | 32 | Substitua as informações com as suas credenciais. 33 | 34 | ### Iniciando a Aplicação 35 | 36 | ``` 37 | docker-compose up --build 38 | ``` 39 | 40 | Isso vai construir os containers e iniciar a aplicação. 41 | 42 | ### Executando Migrações com Alembic 43 | 44 | ``` 45 | docker-compose exec web alembic revision --autogenerate -m "Initial migration" 46 | docker-compose exec web alembic upgrade head 47 | ``` 48 | 49 | ### Acessando os Containers 50 | 51 | ``` 52 | docker ps 53 | docker exec -it 7d11ccb5759a /bin/bash 54 | docker exec -it 18f535251ccb -U myuser -d mydatabase 55 | ``` 56 | 57 | ### Executando os Testes 58 | 59 | ``` 60 | python -m unittest tests.test_app 61 | ``` 62 | 63 | ### Uso 64 | 65 | A aplicação estará disponível em `http://127.0.0.1:8000`. 66 | 67 | ### Documentação 68 | 69 | Você pode acessar a documentação automática da API em `http://127.0.0.1:8000/docs`. -------------------------------------------------------------------------------- /alembic/versions/7792e74809ac_initial_migration.py: -------------------------------------------------------------------------------- 1 | """Initial migration 2 | 3 | Revision ID: 7792e74809ac 4 | Revises: 252ea3c4cd46 5 | Create Date: 2024-08-16 05:33:27.772285 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = '7792e74809ac' 16 | down_revision: Union[str, None] = '252ea3c4cd46' 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column('scrapeddata', 'headings', 24 | existing_type=sa.VARCHAR(), 25 | type_=sa.Text(), 26 | existing_nullable=True) 27 | op.alter_column('scrapeddata', 'links', 28 | existing_type=sa.VARCHAR(), 29 | type_=sa.Text(), 30 | existing_nullable=True) 31 | op.alter_column('scrapeddata', 'content', 32 | existing_type=sa.VARCHAR(), 33 | type_=sa.Text(), 34 | existing_nullable=True) 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade() -> None: 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.alter_column('scrapeddata', 'content', 41 | existing_type=sa.Text(), 42 | type_=sa.VARCHAR(), 43 | existing_nullable=True) 44 | op.alter_column('scrapeddata', 'links', 45 | existing_type=sa.Text(), 46 | type_=sa.VARCHAR(), 47 | existing_nullable=True) 48 | op.alter_column('scrapeddata', 'headings', 49 | existing_type=sa.Text(), 50 | type_=sa.VARCHAR(), 51 | existing_nullable=True) 52 | # ### end Alembic commands ### 53 | -------------------------------------------------------------------------------- /app/crud.py: -------------------------------------------------------------------------------- 1 | from .utils import model_to_dict, scrape_website 2 | from sqlalchemy.ext.asyncio import AsyncSession 3 | from sqlalchemy.exc import SQLAlchemyError 4 | from typing import Optional, List, Dict 5 | from sqlalchemy.future import select 6 | from .database import async_session 7 | from .models import ScrapedData 8 | 9 | async def store_scraped_data(data: Dict[str, Optional[List[str]]], session: AsyncSession): 10 | try: 11 | async with async_session() as session: 12 | scraped_data = ScrapedData( 13 | title=data['title'], 14 | meta_description=data.get('meta_description'), 15 | headings=data.get('headings', []), # Armazena diretamente como lista 16 | links=data.get('links', []), # Armazena diretamente como lista 17 | content=data.get('content', []) # Armazena diretamente como lista 18 | ) 19 | 20 | session.add(scraped_data) 21 | await session.commit() 22 | await session.refresh(scraped_data) 23 | 24 | print("Data stored successfully") 25 | 26 | except SQLAlchemyError as e: 27 | print(f"SQLAlchemy error: {e}") 28 | await session.rollback() 29 | except Exception as e: 30 | print(f"Error while storing data: {e}") 31 | await session.rollback() 32 | 33 | async def scrape_and_store(url: str, session: AsyncSession): 34 | try: 35 | data = scrape_website(url) 36 | await store_scraped_data(data, session) 37 | except Exception as e: 38 | # Adiciona tratamento de exceção adequado 39 | print(f"Error while scraping and storing data: {e}") 40 | 41 | async def read_scraped_data(session: AsyncSession) -> List[Dict[str, Optional[str]]]: 42 | result = await session.execute(select(ScrapedData)) 43 | scraped_data = result.scalars().all() 44 | 45 | return [ 46 | { 47 | "id": data.id, 48 | "title": data.title, 49 | "meta_description": data.meta_description, 50 | "headings": data.headings, 51 | "links": data.links, 52 | "content": data.content 53 | } 54 | for data in scraped_data 55 | ] -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from .schemas import SongCreate, EmailRequest, ScrapeRequest 2 | from .database import engine, Base, async_session 3 | from sqlalchemy.ext.asyncio import AsyncSession 4 | from fastapi import FastAPI, BackgroundTasks 5 | from .utils import send_email, validate_url 6 | from fastapi import HTTPException, status 7 | from sqlalchemy.future import select 8 | from .models import Song 9 | from typing import List 10 | from . import crud 11 | 12 | app = FastAPI() 13 | 14 | # Criando as tabelas de forma assíncrona 15 | async def init_models(): 16 | async with engine.begin() as conn: 17 | await conn.run_sync(Base.metadata.create_all) 18 | 19 | @app.on_event("startup") 20 | async def on_startup(): 21 | await init_models() 22 | 23 | @app.get("/") 24 | async def read_root(): 25 | return {"Hello": "World"} 26 | 27 | @app.post("/songs/", response_model=Song) 28 | async def create_song(song: SongCreate): 29 | async with async_session() as session: 30 | db_song = Song.from_orm(song) 31 | session.add(db_song) 32 | await session.commit() 33 | await session.refresh(db_song) 34 | return db_song 35 | 36 | @app.get("/songs/", response_model=List[Song]) 37 | async def read_songs(): 38 | async with async_session() as session: 39 | result = await session.execute(select(Song)) 40 | songs = result.scalars().all() 41 | return songs 42 | 43 | @app.post("/send-email/") 44 | async def send_email_endpoint(request: EmailRequest, background_tasks: BackgroundTasks): 45 | # Adiciona a tarefa de enviar e-mail em segundo plano 46 | background_tasks.add_task(send_email, request.email, request.subject, request.message) 47 | return {"message": "Email is being sent in the background"} 48 | 49 | @app.post("/scrape/") 50 | async def scrape_endpoint(request: ScrapeRequest, background_tasks: BackgroundTasks): 51 | url = request.url 52 | 53 | # Valida o formato da URL 54 | if not validate_url(url): 55 | raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid URL format") 56 | 57 | async with async_session() as session: 58 | # Inicia a tarefa em segundo plano 59 | background_tasks.add_task(crud.scrape_and_store, url, session) 60 | 61 | return {"message": "Scraping started in the background"} 62 | 63 | @app.get("/scraped-data/") 64 | async def read_scraped_data(): 65 | async with async_session() as session: 66 | data = await crud.read_scraped_data(session) 67 | return data -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import requests 3 | from multiprocessing import Process 4 | from app.main import app 5 | import uvicorn 6 | 7 | # Função para rodar o servidor FastAPI em segundo plano 8 | def start_server(): 9 | uvicorn.run(app, host="127.0.0.1", port=8080) 10 | 11 | class TestApp(unittest.TestCase): 12 | 13 | @classmethod 14 | def setUpClass(cls): 15 | # Inicia o servidor FastAPI em um processo separado 16 | cls.server_process = Process(target=start_server) 17 | cls.server_process.start() 18 | 19 | @classmethod 20 | def tearDownClass(cls): 21 | # Finaliza o servidor FastAPI após os testes 22 | cls.server_process.terminate() 23 | cls.server_process.join() 24 | 25 | def test_read_root(self): 26 | response = requests.get("http://127.0.0.1:8000/") 27 | self.assertEqual(response.status_code, 200) 28 | self.assertEqual(response.json(), {"Hello": "World"}) 29 | 30 | def test_create_song(self): 31 | new_song = { 32 | "name": "Test Song", 33 | "artist": "Test Artist", 34 | "year": 2024 35 | } 36 | response = requests.post("http://127.0.0.1:8000/songs/", json=new_song) 37 | self.assertEqual(response.status_code, 200) 38 | self.assertEqual(response.json()["name"], new_song["name"]) 39 | 40 | def test_read_songs(self): 41 | response = requests.get("http://127.0.0.1:8000/songs/") 42 | self.assertEqual(response.status_code, 200) 43 | self.assertIsInstance(response.json(), list) 44 | 45 | def test_send_email_endpoint(self): 46 | email_request = { 47 | "email": "test@example.com", 48 | "subject": "Test Subject", 49 | "message": "Test Message" 50 | } 51 | response = requests.post("http://127.0.0.1:8000/send-email/", json=email_request) 52 | self.assertEqual(response.status_code, 200) 53 | self.assertEqual(response.json(), {"message": "Email is being sent in the background"}) 54 | 55 | def test_scrape_endpoint(self): 56 | scrape_request = { 57 | "url": "http://example.com" 58 | } 59 | response = requests.post("http://127.0.0.1:8000/scrape/", json=scrape_request) 60 | self.assertEqual(response.status_code, 200) 61 | self.assertEqual(response.json(), {"message": "Scraping started in the background"}) 62 | 63 | def test_read_scraped_data(self): 64 | response = requests.get("http://127.0.0.1:8000/scraped-data/") 65 | self.assertEqual(response.status_code, 200) 66 | self.assertIsInstance(response.json(), list) 67 | 68 | if __name__ == "__main__": 69 | unittest.main() -------------------------------------------------------------------------------- /alembic/env.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from logging.config import fileConfig 3 | 4 | from sqlalchemy import pool 5 | from sqlalchemy.engine import Connection 6 | from sqlalchemy.ext.asyncio import async_engine_from_config 7 | from sqlmodel import SQLModel # NEW 8 | 9 | 10 | from alembic import context 11 | 12 | from app.models import Song # NEW 13 | from sqlmodel import SQLModel 14 | 15 | # this is the Alembic Config object, which provides 16 | # access to the values within the .ini file in use. 17 | config = context.config 18 | 19 | # Interpret the config file for Python logging. 20 | # This line sets up loggers basically. 21 | if config.config_file_name is not None: 22 | fileConfig(config.config_file_name) 23 | 24 | # add your model's MetaData object here 25 | # for 'autogenerate' support 26 | # from myapp import mymodel 27 | # target_metadata = mymodel.Base.metadata 28 | target_metadata = SQLModel.metadata # UPDATED 29 | 30 | # other values from the config, defined by the needs of env.py, 31 | # can be acquired: 32 | # my_important_option = config.get_main_option("my_important_option") 33 | # ... etc. 34 | 35 | 36 | def run_migrations_offline() -> None: 37 | """Run migrations in 'offline' mode. 38 | 39 | This configures the context with just a URL 40 | and not an Engine, though an Engine is acceptable 41 | here as well. By skipping the Engine creation 42 | we don't even need a DBAPI to be available. 43 | 44 | Calls to context.execute() here emit the given string to the 45 | script output. 46 | 47 | """ 48 | url = config.get_main_option("sqlalchemy.url") 49 | context.configure( 50 | url=url, 51 | target_metadata=target_metadata, 52 | literal_binds=True, 53 | dialect_opts={"paramstyle": "named"}, 54 | ) 55 | 56 | with context.begin_transaction(): 57 | context.run_migrations() 58 | 59 | 60 | def do_run_migrations(connection: Connection) -> None: 61 | context.configure(connection=connection, target_metadata=target_metadata) 62 | 63 | with context.begin_transaction(): 64 | context.run_migrations() 65 | 66 | 67 | async def run_async_migrations() -> None: 68 | """In this scenario we need to create an Engine 69 | and associate a connection with the context. 70 | 71 | """ 72 | 73 | connectable = async_engine_from_config( 74 | config.get_section(config.config_ini_section, {}), 75 | prefix="sqlalchemy.", 76 | poolclass=pool.NullPool, 77 | ) 78 | 79 | async with connectable.connect() as connection: 80 | await connection.run_sync(do_run_migrations) 81 | 82 | await connectable.dispose() 83 | 84 | 85 | def run_migrations_online() -> None: 86 | """Run migrations in 'online' mode.""" 87 | 88 | asyncio.run(run_async_migrations()) 89 | 90 | 91 | if context.is_offline_mode(): 92 | run_migrations_offline() 93 | else: 94 | run_migrations_online() -------------------------------------------------------------------------------- /app/utils.py: -------------------------------------------------------------------------------- 1 | from email.mime.multipart import MIMEMultipart 2 | from typing import Optional, List, Dict 3 | from email.mime.text import MIMEText 4 | from dotenv import load_dotenv 5 | from bs4 import BeautifulSoup 6 | from sqlmodel import SQLModel 7 | import requests 8 | import smtplib 9 | import re 10 | import os 11 | 12 | load_dotenv() 13 | 14 | EMAIL = os.getenv("EMAIL") 15 | EMAIL_PASSWORD = os.getenv("EMAIL_PASSWORD") 16 | 17 | def validate_url(url: str) -> bool: 18 | # Verifica se a URL tem um formato válido 19 | url_regex = re.compile( 20 | r'^(?:http|ftp)s?://' # http:// ou https:// 21 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domínio... 22 | r'localhost|' # localhost... 23 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...ou endereço IP 24 | r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...ou IPv6 25 | r'(?::\d+)?' # porta opcional 26 | r'(?:/?|[/?]\S+)$', re.IGNORECASE) 27 | 28 | return re.match(url_regex, url) is not None 29 | 30 | def send_email(email: str, subject: str, message: str): 31 | try: 32 | # Configurações para o Outlook SMTP 33 | smtp_server = "smtp.office365.com" 34 | smtp_port = 587 # Porta para TLS 35 | smtp_user = EMAIL 36 | smtp_password = EMAIL_PASSWORD 37 | 38 | # Criando a mensagem de e-mail 39 | msg = MIMEMultipart() 40 | msg['From'] = smtp_user 41 | msg['To'] = email 42 | msg['Subject'] = subject # Usando o assunto fornecido na requisição 43 | 44 | # Anexando a mensagem ao corpo do email 45 | msg.attach(MIMEText(message, 'plain', 'utf-8')) 46 | 47 | # Configuração do servidor SMTP 48 | with smtplib.SMTP(smtp_server, smtp_port) as server: 49 | server.starttls() # Inicia a criptografia TLS 50 | server.login(smtp_user, smtp_password) 51 | server.sendmail(smtp_user, email, msg.as_string()) 52 | print("Email enviado com sucesso!") 53 | except Exception as e: 54 | print(f"Erro ao enviar e-mail: {e}") 55 | 56 | def model_to_dict(model: SQLModel) -> Dict[str, Optional[List[str]]]: 57 | return {column.name: getattr(model, column.name) for column in model.__table__.columns} 58 | 59 | def scrape_website(url: str) -> Dict[str, Optional[List[str]]]: 60 | response = requests.get(url) 61 | soup = BeautifulSoup(response.content, 'html.parser') 62 | 63 | title = soup.title.string if soup.title else None 64 | meta_description = soup.find('meta', attrs={'name': 'description'}) 65 | meta_description = meta_description['content'] if meta_description else None 66 | 67 | headings = [heading.get_text() for heading in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])] 68 | links = [link['href'] for link in soup.find_all('a', href=True)] 69 | content = [p.get_text() for p in soup.find_all('p')] 70 | 71 | return { 72 | 'title': title, 73 | 'meta_description': meta_description, 74 | 'headings': headings, 75 | 'links': links, 76 | 'content': content 77 | } -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | # Use forward slashes (/) also on windows to provide an os agnostic path 6 | script_location = alembic 7 | 8 | # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s 9 | # Uncomment the line below if you want the files to be prepended with date and time 10 | # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file 11 | # for all available tokens 12 | # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s 13 | 14 | # sys.path path, will be prepended to sys.path if present. 15 | # defaults to the current working directory. 16 | prepend_sys_path = . 17 | 18 | # timezone to use when rendering the date within the migration file 19 | # as well as the filename. 20 | # If specified, requires the python>=3.9 or backports.zoneinfo library. 21 | # Any required deps can installed by adding `alembic[tz]` to the pip requirements 22 | # string value is passed to ZoneInfo() 23 | # leave blank for localtime 24 | # timezone = 25 | 26 | # max length of characters to apply to the "slug" field 27 | # truncate_slug_length = 40 28 | 29 | # set to 'true' to run the environment during 30 | # the 'revision' command, regardless of autogenerate 31 | # revision_environment = false 32 | 33 | # set to 'true' to allow .pyc and .pyo files without 34 | # a source .py file to be detected as revisions in the 35 | # versions/ directory 36 | # sourceless = false 37 | 38 | # version location specification; This defaults 39 | # to alembic/versions. When using multiple version 40 | # directories, initial revisions must be specified with --version-path. 41 | # The path separator used here should be the separator specified by "version_path_separator" below. 42 | # version_locations = %(here)s/bar:%(here)s/bat:alembic/versions 43 | 44 | # version path separator; As mentioned above, this is the character used to split 45 | # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. 46 | # If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. 47 | # Valid values for version_path_separator are: 48 | # 49 | # version_path_separator = : 50 | # version_path_separator = ; 51 | # version_path_separator = space 52 | version_path_separator = os # Use os.pathsep. Default configuration used for new projects. 53 | 54 | # set to 'true' to search source files recursively 55 | # in each "version_locations" directory 56 | # new in Alembic version 1.10 57 | # recursive_version_locations = false 58 | 59 | # the output encoding used when revision files 60 | # are written from script.py.mako 61 | # output_encoding = utf-8 62 | 63 | sqlalchemy.url = postgresql+asyncpg://myuser:mypassword@db:5432/mydatabase 64 | 65 | [post_write_hooks] 66 | # post_write_hooks defines scripts or Python functions that are run 67 | # on newly generated revision scripts. See the documentation for further 68 | # detail and examples 69 | 70 | # format using "black" - use the console_scripts runner, against the "black" entrypoint 71 | # hooks = black 72 | # black.type = console_scripts 73 | # black.entrypoint = black 74 | # black.options = -l 79 REVISION_SCRIPT_FILENAME 75 | 76 | # lint with attempts to fix using "ruff" - use the exec runner, execute a binary 77 | # hooks = ruff 78 | # ruff.type = exec 79 | # ruff.executable = %(here)s/.venv/bin/ruff 80 | # ruff.options = --fix REVISION_SCRIPT_FILENAME 81 | 82 | # Logging configuration 83 | [loggers] 84 | keys = root,sqlalchemy,alembic 85 | 86 | [handlers] 87 | keys = console 88 | 89 | [formatters] 90 | keys = generic 91 | 92 | [logger_root] 93 | level = WARN 94 | handlers = console 95 | qualname = 96 | 97 | [logger_sqlalchemy] 98 | level = WARN 99 | handlers = 100 | qualname = sqlalchemy.engine 101 | 102 | [logger_alembic] 103 | level = INFO 104 | handlers = 105 | qualname = alembic 106 | 107 | [handler_console] 108 | class = StreamHandler 109 | args = (sys.stderr,) 110 | level = NOTSET 111 | formatter = generic 112 | 113 | [formatter_generic] 114 | format = %(levelname)-5.5s [%(name)s] %(message)s 115 | datefmt = %H:%M:%S --------------------------------------------------------------------------------