diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index 4573e2f..b04c759 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -28,6 +28,7 @@ abstração abstraído absurdventures acabar +acabou ação accela Accela @@ -47,12 +48,15 @@ Aceable aceita aceitar aceitas +aceitava aceitável Aceite acentos acentria Acentria +Acessando Acessibilidade +acessíveis acessível acesso acessos @@ -60,6 +64,7 @@ achar acidental acilearning aciona +Acione acluinternships aclunc acog @@ -92,6 +97,7 @@ Adelphi adelphigrouplimited adelphiresearch adequa +adequadamente adfinternational adias adicionada @@ -128,6 +134,7 @@ aestudio AEVEX aevexaerospace afetada +afetado affinidi Affinidi affinitiv @@ -257,6 +264,7 @@ altera alteração alterar alternativa +Alternativas alternativo altium Altium @@ -386,6 +394,7 @@ aplicável apogeetherapeutics apolloio apontando +apontar aponte Apothe apothecom @@ -554,6 +563,7 @@ atualização atualizações atualizada atualizadas +Atualize atualmente Atwell atwellgroup @@ -575,6 +585,7 @@ Austrália autentica autenticada autenticados +autenticam authenticbrandsgroup authenticinsurance autods @@ -590,6 +601,7 @@ automática automaticamente automático automatizadas +automatizados automatticcareers automind Automind @@ -969,6 +981,7 @@ cadência cadrehospice cadvisor cafortune +cair cais CAIS caixa @@ -1233,6 +1246,8 @@ clientes climateai climatecabinet clockworksystems +Clonando +clonar cloudbeds Cloudbeds cloudbedsthirdpartyboard @@ -1386,6 +1401,7 @@ consensys Consensys considere consistência +consolidada consolidadas consomem constantcontact @@ -1510,6 +1526,7 @@ crfamilyofcompanies criação criada criadas +criados criamos Criei Criem @@ -1523,6 +1540,7 @@ crítica criticalmass criticalmassgroup críticas +crítico críticos crmbonus crocodilecloth @@ -1660,6 +1678,7 @@ denverbroncosteamllc departamento Departamentos depende +dependem dependência dependendo depender @@ -1723,6 +1742,7 @@ dhpace diabolocom Diabolocom diacríticos +Dialeto dialpad Dialpad dianahealth @@ -1784,6 +1804,7 @@ distrito distrokid Divco divcowest +divergir diversas dkatalislabs dkbcodefactory @@ -1799,6 +1820,7 @@ Docugami documentação documentadas documentado +documentados Documentar Dodgshun dodgshunmedlin @@ -1883,6 +1905,7 @@ edição edita editadas editados +Edite edmentum Edmentum edobestsandbox @@ -2067,6 +2090,7 @@ escopo escreve escrever escribers +escrito escritorio Escritórios Escuro @@ -2092,6 +2116,7 @@ espelhando espelhar espera esperada +esperadas esperado esperados esperar @@ -2099,6 +2124,7 @@ espirita Espirita esqueci essa +essas essenceit essencialnutricao estabilidade @@ -2164,6 +2190,7 @@ Everlaw everway Everway evgspecialtynetwork +Evidências evio Evio evismart @@ -2188,8 +2215,10 @@ Exames exatamente exati Exati +exatos excecao exceções +excedentes Excelente excelsportsmanagement Excluir @@ -2203,6 +2232,7 @@ executar exemplos exibida exibido +exibidos Exibindo exige exigem @@ -2490,8 +2520,10 @@ fulano Fulano funciona funcional +Funcionalidade funcionalidades funcionando +funcionar funga Funga funil @@ -2573,6 +2605,7 @@ Gerdau gerencia gerenciadas gerenciado +gerenciador Gerenciando gerenciar Gerente @@ -2966,6 +2999,7 @@ inchargeenergy incluem inclui incluído +incluir incode incognia incompleto @@ -3040,6 +3074,7 @@ innodatainc innogames innoveahub insere +inserir insiderstore inspecionam inspiraeducation @@ -3507,9 +3542,11 @@ Mantemos mantendo Mantenha manter +mantida mantido mantl mantrahealth +manuais manualmente manychat Mapa @@ -3651,6 +3688,7 @@ midihealth midpenhousing midpointmarkets mightynetworks +migração Migrações milhares milissegundos @@ -3736,6 +3774,7 @@ monsterenergy monstro montá montada +montadas montar monumentalsports monzo @@ -3804,6 +3843,7 @@ naturesbakery naughtydog navapbc navegável +Navegue navierboat navtechnologies navvis @@ -3943,7 +3983,11 @@ obexp objetivo obrienveterinarygroup obrigatória +obrigatórias +Obrigatórias obrigatório +obrigatórios +Obrigatórios observabilidade Observação Observações @@ -4104,6 +4148,7 @@ pacificlegalfoundation packardculliganwater pacnyc pacote +pacotes Pactual pacvue padronizar @@ -4134,6 +4179,7 @@ panthalassa pantheonpublic pantherlabs papéis +papel paperlessparts papo parachutehealth @@ -4218,6 +4264,7 @@ permissivo permite permitida permitidas +permitido permitidos permitindo permitir @@ -4225,6 +4272,7 @@ perpay perscholashires persefoniaiinc persiste +persistem persistência persistente persistentes @@ -4365,6 +4413,7 @@ precisionmedicinegroup precisionvehicleholdings preciso predictiveindex +Preencha preenche Preenchendo preenchida @@ -4410,6 +4459,7 @@ Priner printou Prioridade Priorização +priorize pris Pris privateequityinsights @@ -4470,6 +4520,7 @@ proteinqureinc protillionbiosciences protonai Protótipo +provedores provisionados próximas próximo @@ -4534,6 +4585,7 @@ quberesearchandtechnologies Quebrada quebrado quebrados +quebrar queracomputinginc Quero querodelivery @@ -4592,6 +4644,7 @@ rebag rebelliondefense rebtel rebuildmanufacturing +recarregar recebe recebendo receber @@ -4628,6 +4681,7 @@ Recurso recursos recusou redcellpartners +redirecionam redpartners redpeak reduz @@ -4639,6 +4693,7 @@ redwoodsoftware Reescrever refatorado referência +Referências referralsuseonly refletem refugeerights @@ -4649,6 +4704,7 @@ registros regra regras Regressão +regressões regscale reidopitaco reindexação @@ -4675,6 +4731,7 @@ relaygraduateschoolofeducation relaypayments relaypro relaytherapeutics +relevante relevantes relishworks relname @@ -4715,6 +4772,7 @@ repositórios Representação reprocessamento reproductivefreedomforall +reproduzir reprofreedomforallinternships repropaga requisições @@ -4821,6 +4879,7 @@ rotacionado rotativo rotativos Roteamento +roteiro rothesaygraduates rothesaylife rslave @@ -4893,6 +4952,7 @@ seattlesoundersfc seazone Seazone Seção +Seções secondharvest secreta secretariatadvisorsllc @@ -5243,6 +5303,7 @@ suficiente sufixo sugerido sugeridos +suíte sujar sumário sumir @@ -5266,6 +5327,7 @@ Superlógica supersod suportar Suporte +suposição supplyhouse supportingstrategies surefirecyber @@ -5387,6 +5449,7 @@ terzo tesseratherapeutics testall testamos +Testando testar teste testlio @@ -5519,6 +5582,7 @@ transmarketgroup trás trase trata +tratado tratados tratam tratamento @@ -5750,8 +5814,10 @@ veristainc veritasvetpartners verkada verramobility +Versão versaterm versionada +versionadas versionado versionados versionar diff --git a/.env.example b/.env.example index cde3e25..c2cc788 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,8 @@ CACHE_TTL_MS=600000 REDIS_KEY_PREFIX=vagas-full KEYWORDS_REDIS_KEY=vagas-full:keywords KEYWORDS_STORAGE_MODE=env +# Mantém GET /keywords ativo, mas bloqueia POST /keywords e publicação no kwsync. +KWSYNC_ENABLED=false # Legacy flags kept for compatibility HEADLESS=false diff --git a/LOCAL_DEVELOPMENT.md b/LOCAL_DEVELOPMENT.md index 315c6ac..01860a7 100644 --- a/LOCAL_DEVELOPMENT.md +++ b/LOCAL_DEVELOPMENT.md @@ -234,13 +234,13 @@ Como criar usuário para testes: ## 7.1 Subir stack completa (recomendado para onboarding) -1. Criar rede: +1; Criar rede: ```bash docker network create vagas-net ``` -2. Subir infra + app + migrate: +2; Subir infra + app + migrate: ```bash docker compose -f docker-compose.infra.yml -f docker-compose.yml -f docker-compose.migrate.yml up --build -d @@ -298,10 +298,10 @@ Use o comando da seção de Docker. Portas esperadas: -- frontend: http://localhost:5173 -- front_admin: http://localhost:5174 -- backend: http://localhost:3001 -- scraper-go: http://localhost:8081 +- frontend: +- front_admin: +- backend: +- scraper-go: ## Caminho B: Node local (frontend + backend) @@ -345,27 +345,27 @@ Observação importante para o Caminho B: URLs principais: -- App principal: http://localhost:5173 -- Login: http://localhost:5173/login -- Cadastro: http://localhost:5173/register +- App principal: +- Login: +- Cadastro: - Dashboard app: /home, /dashboard, /vagas, /mentoria, /perfil, /ajuda - Callback OAuth: /auth/callback Backend: -- Health: http://localhost:3001/health -- Swagger: http://localhost:3001/docs -- Metrics: http://localhost:3001/metrics +- Health: +- Swagger: +- Metrics: Scraper: -- Health: http://localhost:8081/health -- Metrics: http://localhost:8081/metrics -- Admin jobs count: http://localhost:8081/admin/jobs/count +- Health: +- Metrics: +- Admin jobs count: Front admin: -- http://localhost:5174 +- - rota de login: /login - rotas principais: /dashboard, /users, /scrapers, /observability, /audit, /permissions, /settings @@ -430,7 +430,7 @@ Abaixo, os testes manuais sugeridos para os módulos principais. Passos: -1. Acesse http://localhost:5173/login +1. Acesse 2. Tente enviar vazio 3. Informe credenciais inválidas 4. Informe credenciais válidas @@ -445,7 +445,7 @@ Resultado esperado: Passos: -1. Acesse http://localhost:5173/register +1. Acesse 2. Preencha campos obrigatórios 3. Teste telefone opcional vazio 4. Teste telefone válido @@ -500,7 +500,7 @@ Resultado esperado: Passos: -1. Acesse http://localhost:5174/login +1. Acesse 2. Faça login com conta com permissão 3. Navegue por dashboard/users/scrapers/observability/audit/permissions/settings diff --git a/backend/.env.example b/backend/.env.example index ff6615a..37335da 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -38,6 +38,8 @@ CORS_ALLOWED_ORIGINS=http://localhost:5173,http://localhost:5174 DATABASE_URL=postgresql://vagas:vagas@localhost:5432/vagas VALKEY_URL=redis://localhost:6379/0 CACHE_TTL_MS=600000 +# Mantém GET /keywords ativo, mas bloqueia POST /keywords e publicação no kwsync. +KWSYNC_ENABLED=false # Scraping behavior WAIT_BETWEEN_SEARCHES_MS=5000 diff --git a/backend/src/config.ts b/backend/src/config.ts index 59eb0ab..7ebf7ea 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -22,6 +22,7 @@ export interface AppConfig { emailFromAddress: string; emailFromName: string; emailQueueAttempts: number; + kwsyncEnabled: boolean; } function parseBoolean(value: string | undefined, fallback: boolean): boolean { @@ -75,5 +76,6 @@ export function getConfig(): AppConfig { emailFromAddress: process.env.EMAIL_FROM_ADDRESS?.trim() ?? "", emailFromName: process.env.EMAIL_FROM_NAME?.trim() ?? "", emailQueueAttempts: parseNumber(process.env.EMAIL_QUEUE_ATTEMPTS, 3), + kwsyncEnabled: parseBoolean(process.env.KWSYNC_ENABLED, false), }; } diff --git a/backend/src/lib/kwsync.ts b/backend/src/lib/kwsync.ts index 200cb7e..05033b5 100644 --- a/backend/src/lib/kwsync.ts +++ b/backend/src/lib/kwsync.ts @@ -1,4 +1,5 @@ import type { RedisClientType } from "redis"; +import { getConfig } from "../config"; import { logger } from "../logger"; // Chave absoluta global. O Go lerá exatamente esse namespace. @@ -21,6 +22,14 @@ export async function publish( source: KeywordEvent["source"] = "user", userId?: string, ): Promise { + if (!getConfig().kwsyncEnabled) { + logger.info( + { keyword, source, userId }, + "kwsync: publicação ignorada porque KWSYNC_ENABLED=false", + ); + return; + } + const event: KeywordEvent = { keyword, source, @@ -53,6 +62,13 @@ export async function publishBatch( source: KeywordEvent["source"] = "user", ): Promise { if (keywords.length === 0) return; + if (!getConfig().kwsyncEnabled) { + logger.info( + { count: keywords.length, source }, + "kwsync: lote ignorado porque KWSYNC_ENABLED=false", + ); + return; + } const now = new Date().toISOString(); const payloads = keywords.map((keyword) => diff --git a/backend/src/routes/keywords.routes.ts b/backend/src/routes/keywords.routes.ts index 5837df4..23d1d8f 100644 --- a/backend/src/routes/keywords.routes.ts +++ b/backend/src/routes/keywords.routes.ts @@ -4,6 +4,7 @@ import { keywords } from "../db/schema"; import { ownedBy } from "../lib/authorization/ownership"; import { getCache } from "../lib/cache"; import { publish } from "../lib/kwsync"; +import { getConfig } from "../config"; export const keywordsRoutes = Router(); @@ -55,6 +56,8 @@ keywordsRoutes.get("/", async (req, res) => { * responses: * 202: * description: Keyword enfileirada — o Go decide se persiste + * 403: + * description: Submissão de keywords por usuário desabilitada * 400: * description: Dados inválidos */ @@ -62,6 +65,13 @@ keywordsRoutes.post("/", async (req, res) => { const userId = req.session?.userId; if (!userId) return res.status(401).json({ message: "Não autenticado." }); + if (!getConfig().kwsyncEnabled) { + return res.status(403).json({ + ok: false, + message: "Submissão de keywords por usuário está desabilitada.", + }); + } + const raw = req.body?.keyword; const keyword = typeof raw === "string" ? raw.trim() : ""; diff --git a/backend/tests/unit/app.test.ts b/backend/tests/unit/app.test.ts index 0f0c457..0993f6b 100644 --- a/backend/tests/unit/app.test.ts +++ b/backend/tests/unit/app.test.ts @@ -122,6 +122,7 @@ const DEFAULT_PAGINATED = (ids: string[]) => ({ describe("jobsApiApp", () => { beforeEach(() => { vi.clearAllMocks(); + process.env.KWSYNC_ENABLED = "false"; mocks.parsePagination.mockReturnValue(DEFAULT_PAGINATION); mocks.paginate.mockImplementation((ids: string[]) => @@ -709,6 +710,8 @@ describe("jobsApiApp", () => { }); it("POST /keywords enfileira keyword e retorna 202", async () => { + process.env.KWSYNC_ENABLED = "true"; + const app = createJobsApiApp(); const res = await request(app) .post("/keywords") @@ -733,7 +736,26 @@ describe("jobsApiApp", () => { }); }); + it("POST /keywords retorna 403 quando kwsync está desabilitado", async () => { + const app = createJobsApiApp(); + + const res = await request(app) + .post("/keywords") + .send({ keyword: "Rust" }) + .expect(403); + + expect(res.body).toEqual({ + ok: false, + message: "Submissão de keywords por usuário está desabilitada.", + }); + expect(mocks.dbInsert).not.toHaveBeenCalled(); + expect(mocks.getCache).not.toHaveBeenCalled(); + expect(mocks.publish).not.toHaveBeenCalled(); + }); + it("POST /keywords retorna 400 quando keyword está ausente", async () => { + process.env.KWSYNC_ENABLED = "true"; + const app = createJobsApiApp(); const res = await request(app).post("/keywords").send({}).expect(400); @@ -745,6 +767,8 @@ describe("jobsApiApp", () => { }); it("POST /keywords retorna 400 quando keyword é string vazia", async () => { + process.env.KWSYNC_ENABLED = "true"; + const app = createJobsApiApp(); const res = await request(app) @@ -758,6 +782,8 @@ describe("jobsApiApp", () => { }); it("POST /keywords retorna 400 quando keyword não é string", async () => { + process.env.KWSYNC_ENABLED = "true"; + // Arrays não são strings — a rota rejeita com 400 se keyword.trim() não existir // ou com 500 se explodir antes. Ajusta a expectativa ao comportamento real da rota: // req.body?.keyword?.trim() em um array retorna undefined → cai no if → 400 diff --git a/backend/tests/unit/libs/kwsync.test.ts b/backend/tests/unit/libs/kwsync.test.ts index fa29c3d..bcc4fcf 100644 --- a/backend/tests/unit/libs/kwsync.test.ts +++ b/backend/tests/unit/libs/kwsync.test.ts @@ -28,7 +28,20 @@ function getCall(client: ReturnType, index = 0) { } describe("publish", () => { - beforeEach(() => vi.clearAllMocks()); + beforeEach(() => { + vi.clearAllMocks(); + process.env.KWSYNC_ENABLED = "true"; + }); + + it("ignora publicação quando KWSYNC_ENABLED=false", async () => { + process.env.KWSYNC_ENABLED = "false"; + + const client = makeClient(); + await publish(client as any, "React"); + + expect(client.lPush).not.toHaveBeenCalled(); + expect(mocks.loggerInfo).toHaveBeenCalledOnce(); + }); it("chama lPush com a chave correta", async () => { const client = makeClient(); @@ -79,7 +92,10 @@ describe("publish", () => { }); describe("publishBatch", () => { - beforeEach(() => vi.clearAllMocks()); + beforeEach(() => { + vi.clearAllMocks(); + process.env.KWSYNC_ENABLED = "true"; + }); it("nao chama lPush quando keywords e array vazio", async () => { const client = makeClient(); @@ -87,6 +103,16 @@ describe("publishBatch", () => { expect(client.lPush).not.toHaveBeenCalled(); }); + it("ignora lote quando KWSYNC_ENABLED=false", async () => { + process.env.KWSYNC_ENABLED = "false"; + + const client = makeClient(); + await publishBatch(client as any, ["Java", "Node.js"]); + + expect(client.lPush).not.toHaveBeenCalled(); + expect(mocks.loggerInfo).toHaveBeenCalledOnce(); + }); + it("chama lPush uma unica vez com array de payloads", async () => { const client = makeClient(); await publishBatch(client as any, ["Java", "Node.js", "Go"]); diff --git a/frontend/tests/unit/new_dashboard/jobs.test.tsx b/frontend/tests/unit/new_dashboard/jobs.test.tsx index 92680b3..30cb84e 100644 --- a/frontend/tests/unit/new_dashboard/jobs.test.tsx +++ b/frontend/tests/unit/new_dashboard/jobs.test.tsx @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { AddJobModal } from "@/domains/new_dashboard/components/jobs/AddJobModal"; import { JobDetailModal } from "@/domains/new_dashboard/components/jobs/JobDetailModal"; import { JobFilter } from "@/domains/new_dashboard/components/jobs/JobFilter"; +import { FormattedJobDescription } from "@/domains/new_dashboard/components/jobs/FormattedJobDescription"; import { JobRow } from "@/domains/new_dashboard/components/jobs/JobRow"; import { JobTab } from "@/domains/new_dashboard/components/jobs/JobTab"; import { JobTable } from "@/domains/new_dashboard/components/jobs/JobTable"; @@ -340,6 +341,55 @@ describe("new_dashboard job components", () => { expect(screen.queryByText(/<h3>/i)).not.toBeInTheDocument(); }); + it("renderiza texto puro, tags semânticas e links inseguros da descrição", () => { + const plainRender = render( + , + ); + + expect(screen.getByText(/linha 1/i)).toBeInTheDocument(); + expect(screen.getByText(/linha 2/i)).toBeInTheDocument(); + expect(plainRender.container.querySelector("p")).toHaveClass( + "whitespace-pre-wrap", + ); + + plainRender.unmount(); + + const { container } = render( + Título principal", + "

Subtítulo

", + "

Grupo

", + "
Citação
", + "
npm test
", + "
", + 'Link bloqueado', + 'Link relativo seguro', + "", + "
Conteúdo preservado
", + ].join("")} + />, + ); + + expect( + screen.getByRole("heading", { name: "Título principal", level: 1 }), + ).toBeInTheDocument(); + expect( + screen.getByRole("heading", { name: "Subtítulo", level: 2 }), + ).toBeInTheDocument(); + expect( + screen.getByRole("heading", { name: "Grupo", level: 4 }), + ).toBeInTheDocument(); + expect(container.querySelector("blockquote")).toHaveTextContent("Citação"); + expect(container.querySelector("pre")).toHaveTextContent("npm test"); + expect(container.querySelector("hr")).toBeInTheDocument(); + expect(screen.getByText("Link bloqueado").tagName).toBe("SPAN"); + expect(screen.getByRole("link", { name: "Link relativo seguro" })) + .toHaveAttribute("href", "http://localhost:3000/vaga"); + expect(container.querySelector("img")).not.toBeInTheDocument(); + expect(screen.getByText("Conteúdo preservado")).toBeInTheDocument(); + }); + it("valida e salva uma vaga manual nova", () => { const onAddJob = vi.fn(); const onClose = vi.fn(); diff --git a/frontend/tests/unit/services/connectionsApi.test.ts b/frontend/tests/unit/services/connectionsApi.test.ts new file mode 100644 index 0000000..24d77c4 --- /dev/null +++ b/frontend/tests/unit/services/connectionsApi.test.ts @@ -0,0 +1,149 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + connectProvider, + disconnectProvider, + getConnections, +} from "@/domains/auth/infrastructure/connectionsApi"; + +const fetchMock = vi.fn(); +globalThis.fetch = fetchMock as any; + +function mockResponse({ + ok = true, + jsonData = {}, + jsonRejects = false, +}: { + ok?: boolean; + jsonData?: unknown; + jsonRejects?: boolean; +} = {}) { + return { + ok, + json: jsonRejects + ? vi.fn().mockRejectedValue(new SyntaxError("invalid json")) + : vi.fn().mockResolvedValue(jsonData), + }; +} + +describe("connectionsApi", () => { + beforeEach(() => { + fetchMock.mockReset(); + vi.unstubAllEnvs(); + vi.stubEnv("VITE_API_BASE_URL", ""); + }); + + it("carrega conexões usando URL base normalizada", async () => { + vi.stubEnv("VITE_API_BASE_URL", "https://api.example.com///"); + fetchMock.mockResolvedValueOnce( + mockResponse({ + jsonData: { + hasPassword: true, + connections: [ + { + provider: "github", + connected: true, + connectedAt: "2026-07-01T12:00:00.000Z", + }, + ], + }, + }), + ); + + await expect(getConnections()).resolves.toEqual({ + hasPassword: true, + connections: [ + { + provider: "github", + connected: true, + connectedAt: "2026-07-01T12:00:00.000Z", + }, + ], + }); + expect(fetchMock).toHaveBeenCalledWith( + "https://api.example.com/auth/connections", + { credentials: "include" }, + ); + }); + + it("lança erro ao falhar no carregamento das conexões", async () => { + fetchMock.mockResolvedValueOnce(mockResponse({ ok: false })); + + await expect(getConnections()).rejects.toThrow( + "Falha ao carregar conexões.", + ); + expect(fetchMock).toHaveBeenCalledWith("/auth/connections", { + credentials: "include", + }); + }); + + it("desconecta provider e usa mensagem do backend quando disponível", async () => { + fetchMock.mockResolvedValueOnce(mockResponse()); + await expect(disconnectProvider("google")).resolves.toBeUndefined(); + + expect(fetchMock).toHaveBeenCalledWith("/auth/connections/google", { + method: "DELETE", + credentials: "include", + }); + + fetchMock.mockResolvedValueOnce( + mockResponse({ + ok: false, + jsonData: { message: "Conexão principal não pode ser removida." }, + }), + ); + + await expect(disconnectProvider("linkedin")).rejects.toThrow( + "Conexão principal não pode ser removida.", + ); + }); + + it("usa mensagem padrão quando erro de desconexão não tem JSON válido", async () => { + fetchMock.mockResolvedValueOnce( + mockResponse({ + ok: false, + jsonRejects: true, + }), + ); + + await expect(disconnectProvider("github")).rejects.toThrow( + "Falha ao desconectar.", + ); + }); + + it("inicia conexão OAuth redirecionando para a URL recebida", async () => { + const originalLocation = window.location; + Object.defineProperty(window, "location", { + configurable: true, + value: { href: "http://localhost/" }, + }); + + fetchMock.mockResolvedValueOnce( + mockResponse({ + jsonData: { url: "https://github.com/login/oauth/authorize" }, + }), + ); + + await connectProvider("github"); + + expect(fetchMock).toHaveBeenCalledWith("/auth/github/url?intent=link", { + credentials: "include", + }); + expect(window.location.href).toBe( + "https://github.com/login/oauth/authorize", + ); + + Object.defineProperty(window, "location", { + configurable: true, + value: originalLocation, + }); + }); + + it("lança erro quando não consegue iniciar conexão OAuth", async () => { + fetchMock.mockResolvedValueOnce(mockResponse({ ok: false })); + + await expect(connectProvider("google")).rejects.toThrow( + "Falha ao iniciar conexão.", + ); + }); +}); diff --git a/scraper-go/Dockerfile b/scraper-go/Dockerfile index 975d20a..86ab878 100644 --- a/scraper-go/Dockerfile +++ b/scraper-go/Dockerfile @@ -22,11 +22,12 @@ WORKDIR /app COPY --from=builder /go-scraper /go-scraper -COPY --from=builder /app/internal/keywords/keywords.json ./internal/keywords/keywords.json +COPY --from=builder /app/internal/keywords/keywords.json ./internal/keywords/keywords.json +COPY --from=builder /app/internal/keywords/generator.json ./internal/keywords/generator.json COPY --from=builder /app/internal/interfaces ./internal/interfaces EXPOSE 8081 ENV GO_SCRAPER_ADDR=:8081 -ENTRYPOINT ["/go-scraper"] \ No newline at end of file +ENTRYPOINT ["/go-scraper"] diff --git a/scraper-go/cmd/server/handlers.go b/scraper-go/cmd/server/handlers.go index 98448aa..37adf66 100644 --- a/scraper-go/cmd/server/handlers.go +++ b/scraper-go/cmd/server/handlers.go @@ -111,7 +111,9 @@ func handleSaveKeywords(kwStore *keywords.Store) http.HandlerFunc { return } - if err := kwStore.Save(r.Context(), body.Keywords); err != nil { + expanded := keywords.GenerateSearchKeywords(body.Keywords) + + if err := kwStore.Save(r.Context(), expanded); err != nil { http.Error(w, "erro ao salvar keywords", http.StatusInternalServerError) return } @@ -119,7 +121,7 @@ func handleSaveKeywords(kwStore *keywords.Store) http.HandlerFunc { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ "ok": true, - "keywords": body.Keywords, + "keywords": expanded, }) } } diff --git a/scraper-go/internal/keywords/defaults.go b/scraper-go/internal/keywords/defaults.go index 02e08c3..3b6140a 100644 --- a/scraper-go/internal/keywords/defaults.go +++ b/scraper-go/internal/keywords/defaults.go @@ -31,7 +31,7 @@ func LoadDefaultKeywords() []string { return []string{} } - normalized := NormalizeKeywords(parsed.Keywords) + normalized := GenerateSearchKeywords(parsed.Keywords) slog.Info("keywords: loaded from file", "path", p, "count", len(normalized)) return normalized } diff --git a/scraper-go/internal/keywords/generator.go b/scraper-go/internal/keywords/generator.go new file mode 100644 index 0000000..7498c7f --- /dev/null +++ b/scraper-go/internal/keywords/generator.go @@ -0,0 +1,306 @@ +package keywords + +import ( + "encoding/json" + "log/slog" + "os" + "sync" +) + +const maxGeneratedCombinations = 200 + +type keywordCategory string + +const ( + categoryBackend keywordCategory = "backend" + categoryFrontend keywordCategory = "frontend" + categoryFullstack keywordCategory = "fullstack" + categoryMobile keywordCategory = "mobile" + categoryData keywordCategory = "data" + categoryDevOps keywordCategory = "devops" + categoryPlatform keywordCategory = "platform" + categoryQA keywordCategory = "qa" + categorySecurity keywordCategory = "security" + categoryCRM keywordCategory = "crm" + categoryERP keywordCategory = "erp" + categoryIntegration keywordCategory = "integration" + categoryBlockchain keywordCategory = "blockchain" + categoryEmbedded keywordCategory = "embedded" + categoryGame keywordCategory = "game" +) + +type titleTerm struct { + name string + categories []keywordCategory +} + +type technologyTerm struct { + name string + categories []keywordCategory +} + +type generatorFile struct { + Titles []generatorTerm `json:"titles"` + Technologies []generatorTerm `json:"technologies"` +} + +type generatorTerm struct { + Name string `json:"name"` + Categories []keywordCategory `json:"categories"` +} + +type generatorData struct { + titles []titleTerm + technologies []technologyTerm +} + +var ( + generatorOnce sync.Once + generatorConfig generatorData +) + +var defaultKeywordTitles = []titleTerm{ + {name: "backend developer", categories: []keywordCategory{categoryBackend}}, + {name: "backend engineer", categories: []keywordCategory{categoryBackend}}, + {name: "frontend developer", categories: []keywordCategory{categoryFrontend}}, + {name: "frontend engineer", categories: []keywordCategory{categoryFrontend}}, + {name: "full stack developer", categories: []keywordCategory{categoryFullstack, categoryBackend, categoryFrontend}}, + {name: "full stack engineer", categories: []keywordCategory{categoryFullstack, categoryBackend, categoryFrontend}}, + {name: "fullstack developer", categories: []keywordCategory{categoryFullstack, categoryBackend, categoryFrontend}}, + {name: "fullstack engineer", categories: []keywordCategory{categoryFullstack, categoryBackend, categoryFrontend}}, + {name: "software engineer", categories: []keywordCategory{categoryBackend, categoryFrontend, categoryFullstack, categoryMobile}}, + {name: "software developer", categories: []keywordCategory{categoryBackend, categoryFrontend, categoryFullstack, categoryMobile}}, + {name: "mobile developer", categories: []keywordCategory{categoryMobile}}, + {name: "mobile engineer", categories: []keywordCategory{categoryMobile}}, + {name: "data engineer", categories: []keywordCategory{categoryData}}, + {name: "data analyst", categories: []keywordCategory{categoryData}}, + {name: "data scientist", categories: []keywordCategory{categoryData}}, + {name: "machine learning engineer", categories: []keywordCategory{categoryData}}, + {name: "devops engineer", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "cloud engineer", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "platform engineer", categories: []keywordCategory{categoryPlatform, categoryDevOps}}, + {name: "infrastructure engineer", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "qa engineer", categories: []keywordCategory{categoryQA}}, + {name: "qa analyst", categories: []keywordCategory{categoryQA}}, + {name: "automation engineer", categories: []keywordCategory{categoryQA}}, + {name: "security engineer", categories: []keywordCategory{categorySecurity}}, + {name: "application security engineer", categories: []keywordCategory{categorySecurity, categoryBackend}}, + {name: "salesforce developer", categories: []keywordCategory{categoryCRM}}, + {name: "salesforce engineer", categories: []keywordCategory{categoryCRM}}, + {name: "salesforce architect", categories: []keywordCategory{categoryCRM}}, + {name: "sap consultant", categories: []keywordCategory{categoryERP}}, + {name: "sap developer", categories: []keywordCategory{categoryERP}}, + {name: "integration developer", categories: []keywordCategory{categoryIntegration, categoryBackend}}, + {name: "integration engineer", categories: []keywordCategory{categoryIntegration, categoryBackend}}, + {name: "blockchain developer", categories: []keywordCategory{categoryBlockchain}}, + {name: "embedded engineer", categories: []keywordCategory{categoryEmbedded}}, + {name: "firmware engineer", categories: []keywordCategory{categoryEmbedded}}, + {name: "game developer", categories: []keywordCategory{categoryGame}}, +} + +var defaultKeywordTechnologies = []technologyTerm{ + {name: "react", categories: []keywordCategory{categoryFrontend, categoryFullstack}}, + {name: "next.js", categories: []keywordCategory{categoryFrontend, categoryFullstack}}, + {name: "vue", categories: []keywordCategory{categoryFrontend, categoryFullstack}}, + {name: "angular", categories: []keywordCategory{categoryFrontend, categoryFullstack}}, + {name: "javascript", categories: []keywordCategory{categoryFrontend, categoryBackend, categoryFullstack}}, + {name: "typescript", categories: []keywordCategory{categoryFrontend, categoryBackend, categoryFullstack}}, + {name: "node.js", categories: []keywordCategory{categoryBackend, categoryFullstack}}, + {name: "nestjs", categories: []keywordCategory{categoryBackend, categoryFullstack}}, + {name: "java", categories: []keywordCategory{categoryBackend}}, + {name: "spring boot", categories: []keywordCategory{categoryBackend}}, + {name: "python", categories: []keywordCategory{categoryBackend, categoryData, categoryQA}}, + {name: "django", categories: []keywordCategory{categoryBackend}}, + {name: "flask", categories: []keywordCategory{categoryBackend}}, + {name: "go", categories: []keywordCategory{categoryBackend}}, + {name: "golang", categories: []keywordCategory{categoryBackend}}, + {name: "php", categories: []keywordCategory{categoryBackend}}, + {name: "laravel", categories: []keywordCategory{categoryBackend}}, + {name: "ruby on rails", categories: []keywordCategory{categoryBackend}}, + {name: ".net", categories: []keywordCategory{categoryBackend}}, + {name: "c#", categories: []keywordCategory{categoryBackend}}, + {name: "react native", categories: []keywordCategory{categoryMobile}}, + {name: "flutter", categories: []keywordCategory{categoryMobile}}, + {name: "android", categories: []keywordCategory{categoryMobile}}, + {name: "ios", categories: []keywordCategory{categoryMobile}}, + {name: "kotlin", categories: []keywordCategory{categoryMobile}}, + {name: "swift", categories: []keywordCategory{categoryMobile}}, + {name: "docker", categories: []keywordCategory{categoryDevOps, categoryPlatform, categoryBackend}}, + {name: "kubernetes", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "terraform", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "ansible", categories: []keywordCategory{categoryDevOps, categoryPlatform}}, + {name: "aws", categories: []keywordCategory{categoryDevOps, categoryPlatform, categoryBackend, categoryData}}, + {name: "azure", categories: []keywordCategory{categoryDevOps, categoryPlatform, categoryBackend, categoryData}}, + {name: "gcp", categories: []keywordCategory{categoryDevOps, categoryPlatform, categoryBackend, categoryData}}, + {name: "postgresql", categories: []keywordCategory{categoryBackend, categoryData}}, + {name: "mysql", categories: []keywordCategory{categoryBackend, categoryData}}, + {name: "mongodb", categories: []keywordCategory{categoryBackend, categoryData}}, + {name: "redis", categories: []keywordCategory{categoryBackend}}, + {name: "kafka", categories: []keywordCategory{categoryBackend, categoryData, categoryIntegration}}, + {name: "rabbitmq", categories: []keywordCategory{categoryBackend, categoryIntegration}}, + {name: "graphql", categories: []keywordCategory{categoryFrontend, categoryBackend, categoryFullstack}}, + {name: "selenium", categories: []keywordCategory{categoryQA}}, + {name: "cypress", categories: []keywordCategory{categoryQA}}, + {name: "playwright", categories: []keywordCategory{categoryQA}}, + {name: "salesforce", categories: []keywordCategory{categoryCRM}}, + {name: "apex", categories: []keywordCategory{categoryCRM}}, + {name: "lwc", categories: []keywordCategory{categoryCRM}}, + {name: "mulesoft", categories: []keywordCategory{categoryIntegration, categoryCRM}}, + {name: "sap abap", categories: []keywordCategory{categoryERP}}, + {name: "sap hana", categories: []keywordCategory{categoryERP}}, + {name: "sap fiori", categories: []keywordCategory{categoryERP, categoryFrontend}}, + {name: "oracle", categories: []keywordCategory{categoryBackend, categoryERP}}, + {name: "power platform", categories: []keywordCategory{categoryCRM}}, + {name: "solidity", categories: []keywordCategory{categoryBlockchain}}, + {name: "web3", categories: []keywordCategory{categoryBlockchain}}, + {name: "unity", categories: []keywordCategory{categoryGame}}, + {name: "unreal engine", categories: []keywordCategory{categoryGame}}, +} + +func GenerateSearchKeywords(raw []string) []string { + base := NormalizeKeywords(raw) + result := make([]string, 0, len(base)+maxGeneratedCombinations) + seen := make(map[string]struct{}, len(base)+maxGeneratedCombinations) + + for _, keyword := range base { + addKeyword(&result, seen, keyword) + } + + cfg := loadGeneratorConfig() + titles := matchingTitles(base, cfg.titles) + technologies := matchingTechnologies(base, cfg.technologies) + evaluatedCombinations := 0 + + for _, title := range titles { + for _, tech := range technologies { + if !categoriesOverlap(title.categories, tech.categories) { + continue + } + if evaluatedCombinations >= maxGeneratedCombinations { + return result + } + evaluatedCombinations++ + addKeyword(&result, seen, tech.name+" "+title.name) + } + } + + return result +} + +func loadGeneratorConfig() generatorData { + generatorOnce.Do(func() { + generatorConfig = generatorData{ + titles: defaultKeywordTitles, + technologies: defaultKeywordTechnologies, + } + + paths := []string{ + "/app/internal/keywords/generator.json", + "./internal/keywords/generator.json", + } + + for _, p := range paths { + data, err := os.ReadFile(p) + if err != nil { + continue + } + + var parsed generatorFile + if err := json.Unmarshal(data, &parsed); err != nil { + slog.Error("keywords: failed to parse generator.json", "path", p, "error", err) + return + } + + titles := make([]titleTerm, 0, len(parsed.Titles)) + for _, term := range parsed.Titles { + if term.Name == "" || len(term.Categories) == 0 { + continue + } + titles = append(titles, titleTerm{name: term.Name, categories: term.Categories}) + } + + technologies := make([]technologyTerm, 0, len(parsed.Technologies)) + for _, term := range parsed.Technologies { + if term.Name == "" || len(term.Categories) == 0 { + continue + } + technologies = append(technologies, technologyTerm{name: term.Name, categories: term.Categories}) + } + + if len(titles) == 0 || len(technologies) == 0 { + slog.Warn("keywords: generator.json vazio, usando fallback interno", "path", p) + return + } + + generatorConfig = generatorData{titles: titles, technologies: technologies} + slog.Info("keywords: generator loaded from file", "path", p, "titles", len(titles), "technologies", len(technologies)) + return + } + }) + + return generatorConfig +} + +func matchingTitles(seed []string, titles []titleTerm) []titleTerm { + found := make([]titleTerm, 0, len(titles)) + for _, title := range titles { + for _, keyword := range seed { + if keyword == title.name { + found = append(found, title) + break + } + } + } + return found +} + +func matchingTechnologies(seed []string, technologies []technologyTerm) []technologyTerm { + found := make([]technologyTerm, 0, len(technologies)) + seen := make(map[string]struct{}, len(technologies)) + + for _, tech := range technologies { + for _, keyword := range seed { + if keywordContainsTerm(keyword, tech.name) { + if _, exists := seen[tech.name]; !exists { + found = append(found, tech) + seen[tech.name] = struct{}{} + } + break + } + } + } + + return found +} + +func keywordContainsTerm(keyword string, term string) bool { + return keyword == term || + keyword == term+" developer" || + keyword == term+" engineer" || + keyword == term+" analyst" || + keyword == term+" consultant" || + keyword == term+" architect" +} + +func categoriesOverlap(left []keywordCategory, right []keywordCategory) bool { + for _, l := range left { + for _, r := range right { + if l == r { + return true + } + } + } + return false +} + +func addKeyword(result *[]string, seen map[string]struct{}, keyword string) bool { + if keyword == "" { + return false + } + if _, exists := seen[keyword]; exists { + return false + } + seen[keyword] = struct{}{} + *result = append(*result, keyword) + return true +} diff --git a/scraper-go/internal/keywords/generator.json b/scraper-go/internal/keywords/generator.json new file mode 100644 index 0000000..09ae168 --- /dev/null +++ b/scraper-go/internal/keywords/generator.json @@ -0,0 +1,98 @@ +{ + "titles": [ + { "name": "backend developer", "categories": ["backend"] }, + { "name": "backend engineer", "categories": ["backend"] }, + { "name": "frontend developer", "categories": ["frontend"] }, + { "name": "frontend engineer", "categories": ["frontend"] }, + { "name": "full stack developer", "categories": ["fullstack", "backend", "frontend"] }, + { "name": "full stack engineer", "categories": ["fullstack", "backend", "frontend"] }, + { "name": "fullstack developer", "categories": ["fullstack", "backend", "frontend"] }, + { "name": "fullstack engineer", "categories": ["fullstack", "backend", "frontend"] }, + { "name": "software engineer", "categories": ["backend", "frontend", "fullstack", "mobile"] }, + { "name": "software developer", "categories": ["backend", "frontend", "fullstack", "mobile"] }, + { "name": "mobile developer", "categories": ["mobile"] }, + { "name": "mobile engineer", "categories": ["mobile"] }, + { "name": "data engineer", "categories": ["data"] }, + { "name": "data analyst", "categories": ["data"] }, + { "name": "data scientist", "categories": ["data"] }, + { "name": "machine learning engineer", "categories": ["data"] }, + { "name": "devops engineer", "categories": ["devops", "platform"] }, + { "name": "cloud engineer", "categories": ["devops", "platform"] }, + { "name": "platform engineer", "categories": ["platform", "devops"] }, + { "name": "infrastructure engineer", "categories": ["devops", "platform"] }, + { "name": "qa engineer", "categories": ["qa"] }, + { "name": "qa analyst", "categories": ["qa"] }, + { "name": "automation engineer", "categories": ["qa"] }, + { "name": "security engineer", "categories": ["security"] }, + { "name": "application security engineer", "categories": ["security", "backend"] }, + { "name": "salesforce developer", "categories": ["crm"] }, + { "name": "salesforce engineer", "categories": ["crm"] }, + { "name": "salesforce architect", "categories": ["crm"] }, + { "name": "sap consultant", "categories": ["erp"] }, + { "name": "sap developer", "categories": ["erp"] }, + { "name": "integration developer", "categories": ["integration", "backend"] }, + { "name": "integration engineer", "categories": ["integration", "backend"] }, + { "name": "blockchain developer", "categories": ["blockchain"] }, + { "name": "embedded engineer", "categories": ["embedded"] }, + { "name": "firmware engineer", "categories": ["embedded"] }, + { "name": "game developer", "categories": ["game"] } + ], + "technologies": [ + { "name": "react", "categories": ["frontend", "fullstack"] }, + { "name": "next.js", "categories": ["frontend", "fullstack"] }, + { "name": "vue", "categories": ["frontend", "fullstack"] }, + { "name": "angular", "categories": ["frontend", "fullstack"] }, + { "name": "javascript", "categories": ["frontend", "backend", "fullstack"] }, + { "name": "typescript", "categories": ["frontend", "backend", "fullstack"] }, + { "name": "node.js", "categories": ["backend", "fullstack"] }, + { "name": "nestjs", "categories": ["backend", "fullstack"] }, + { "name": "java", "categories": ["backend"] }, + { "name": "spring boot", "categories": ["backend"] }, + { "name": "python", "categories": ["backend", "data", "qa"] }, + { "name": "django", "categories": ["backend"] }, + { "name": "flask", "categories": ["backend"] }, + { "name": "go", "categories": ["backend"] }, + { "name": "golang", "categories": ["backend"] }, + { "name": "php", "categories": ["backend"] }, + { "name": "laravel", "categories": ["backend"] }, + { "name": "ruby on rails", "categories": ["backend"] }, + { "name": ".net", "categories": ["backend"] }, + { "name": "c#", "categories": ["backend"] }, + { "name": "react native", "categories": ["mobile"] }, + { "name": "flutter", "categories": ["mobile"] }, + { "name": "android", "categories": ["mobile"] }, + { "name": "ios", "categories": ["mobile"] }, + { "name": "kotlin", "categories": ["mobile"] }, + { "name": "swift", "categories": ["mobile"] }, + { "name": "docker", "categories": ["devops", "platform", "backend"] }, + { "name": "kubernetes", "categories": ["devops", "platform"] }, + { "name": "terraform", "categories": ["devops", "platform"] }, + { "name": "ansible", "categories": ["devops", "platform"] }, + { "name": "aws", "categories": ["devops", "platform", "backend", "data"] }, + { "name": "azure", "categories": ["devops", "platform", "backend", "data"] }, + { "name": "gcp", "categories": ["devops", "platform", "backend", "data"] }, + { "name": "postgresql", "categories": ["backend", "data"] }, + { "name": "mysql", "categories": ["backend", "data"] }, + { "name": "mongodb", "categories": ["backend", "data"] }, + { "name": "redis", "categories": ["backend"] }, + { "name": "kafka", "categories": ["backend", "data", "integration"] }, + { "name": "rabbitmq", "categories": ["backend", "integration"] }, + { "name": "graphql", "categories": ["frontend", "backend", "fullstack"] }, + { "name": "selenium", "categories": ["qa"] }, + { "name": "cypress", "categories": ["qa"] }, + { "name": "playwright", "categories": ["qa"] }, + { "name": "salesforce", "categories": ["crm"] }, + { "name": "apex", "categories": ["crm"] }, + { "name": "lwc", "categories": ["crm"] }, + { "name": "mulesoft", "categories": ["integration", "crm"] }, + { "name": "sap abap", "categories": ["erp"] }, + { "name": "sap hana", "categories": ["erp"] }, + { "name": "sap fiori", "categories": ["erp", "frontend"] }, + { "name": "oracle", "categories": ["backend", "erp"] }, + { "name": "power platform", "categories": ["crm"] }, + { "name": "solidity", "categories": ["blockchain"] }, + { "name": "web3", "categories": ["blockchain"] }, + { "name": "unity", "categories": ["game"] }, + { "name": "unreal engine", "categories": ["game"] } + ] +} diff --git a/scraper-go/internal/keywords/normalize_test.go b/scraper-go/internal/keywords/normalize_test.go index db3e56b..f7bc822 100644 --- a/scraper-go/internal/keywords/normalize_test.go +++ b/scraper-go/internal/keywords/normalize_test.go @@ -2,6 +2,7 @@ package keywords import ( "reflect" + "strconv" "testing" ) @@ -13,3 +14,198 @@ func TestNormalizeKeywordsTrimsLowercasesAndDeduplicates(t *testing.T) { t.Fatalf("NormalizeKeywords() = %#v, want %#v", got, want) } } + +func TestGenerateSearchKeywordsBuildsCompatibleTitleTechnologyQueries(t *testing.T) { + got := GenerateSearchKeywords([]string{ + "Frontend Developer", + "Backend Developer", + "DevOps Engineer", + "React", + "Spring Boot", + "Docker", + }) + + assertContains(t, got, "react frontend developer") + assertContains(t, got, "spring boot backend developer") + assertContains(t, got, "docker devops engineer") + assertNotContains(t, got, "spring boot frontend developer") +} + +func TestGenerateSearchKeywordsKeepsStableLimit(t *testing.T) { + input := []string{ + "frontend developer", + "frontend engineer", + "backend developer", + "backend engineer", + "full stack developer", + "software engineer", + "mobile developer", + "data engineer", + "devops engineer", + "platform engineer", + "qa engineer", + "security engineer", + "react", + "next.js", + "vue", + "angular", + "typescript", + "node.js", + "java", + "spring boot", + "python", + "go", + "docker", + "kubernetes", + "terraform", + "aws", + "postgresql", + "kafka", + "react native", + "flutter", + "selenium", + } + + got := GenerateSearchKeywords(input) + if len(got) > len(NormalizeKeywords(input))+maxGeneratedCombinations { + t.Fatalf("GenerateSearchKeywords() returned %d items, want <= %d", len(got), len(NormalizeKeywords(input))+maxGeneratedCombinations) + } +} + +func TestGenerateSearchKeywordsIsIdempotent(t *testing.T) { + input := []string{ + "backend developer", + "backend engineer", + "frontend developer", + "frontend engineer", + "full stack developer", + "full stack engineer", + "fullstack developer", + "fullstack engineer", + "software engineer", + "software developer", + "mobile developer", + "mobile engineer", + "data engineer", + "data analyst", + "data scientist", + "machine learning engineer", + "devops engineer", + "cloud engineer", + "platform engineer", + "infrastructure engineer", + "qa engineer", + "qa analyst", + "automation engineer", + "security engineer", + "application security engineer", + "salesforce developer", + "salesforce engineer", + "salesforce architect", + "sap consultant", + "sap developer", + "integration developer", + "integration engineer", + "blockchain developer", + "embedded engineer", + "firmware engineer", + "game developer", + "react", + "next.js", + "vue", + "angular", + "javascript", + "typescript", + "node.js", + "nestjs", + "java", + "spring boot", + "python", + "django", + "flask", + "go", + "golang", + "php", + "laravel", + "ruby on rails", + ".net", + "c#", + "react native", + "flutter", + "android", + "ios", + "kotlin", + "swift", + "docker", + "kubernetes", + "terraform", + "ansible", + "aws", + "azure", + "gcp", + "postgresql", + "mysql", + "mongodb", + "redis", + "kafka", + "rabbitmq", + "graphql", + "selenium", + "cypress", + "playwright", + "salesforce", + "apex", + "lwc", + "mulesoft", + "sap abap", + "sap hana", + "sap fiori", + "oracle", + "power platform", + "solidity", + "web3", + "unity", + "unreal engine", + } + + once := GenerateSearchKeywords(input) + twice := GenerateSearchKeywords(once) + + if len(once) > len(NormalizeKeywords(input))+maxGeneratedCombinations { + t.Fatalf("GenerateSearchKeywords() returned %d items, want <= %d", len(once), len(NormalizeKeywords(input))+maxGeneratedCombinations) + } + if !reflect.DeepEqual(twice, once) { + t.Fatalf("GenerateSearchKeywords() should be idempotent; once=%d twice=%d", len(once), len(twice)) + } +} + +func TestGenerateSearchKeywordsPreservesLargeSeedLists(t *testing.T) { + input := make([]string, 0, maxGeneratedCombinations+1) + for i := 0; i < maxGeneratedCombinations+1; i++ { + input = append(input, "keyword "+strconv.Itoa(i)) + } + + got := GenerateSearchKeywords(input) + if len(got) != len(input) { + t.Fatalf("GenerateSearchKeywords() returned %d items, want all %d seed keywords preserved", len(got), len(input)) + } +} + +func assertContains(t *testing.T, values []string, expected string) { + t.Helper() + for _, value := range values { + if value == expected { + return + } + } + t.Fatalf("expected %#v to contain %q", values, expected) +} + +func assertNotContains(t *testing.T, values []string, unexpected string) { + t.Helper() + for _, value := range values { + if value == unexpected { + t.Fatalf("expected %#v not to contain %q", values, unexpected) + } + } +} diff --git a/scraper-go/internal/keywords/store.go b/scraper-go/internal/keywords/store.go index 74f8ef2..05fe2c3 100644 --- a/scraper-go/internal/keywords/store.go +++ b/scraper-go/internal/keywords/store.go @@ -34,13 +34,13 @@ func (s *Store) Load(ctx context.Context) ([]string, error) { return fallback, nil } - return NormalizeKeywords(raw), nil + return GenerateSearchKeywords(raw), nil } // Save persiste as keywords no Valkey sem expiração (TTL=0). // Keywords são configuração, não cache — não devem expirar automaticamente. func (s *Store) Save(ctx context.Context, keywords []string) error { - normalized := NormalizeKeywords(keywords) + normalized := GenerateSearchKeywords(keywords) if err := s.cache.Set(ctx, cacheKey, normalized, 0); err != nil { return err diff --git a/scraper-go/internal/kwsync/kwsync.go b/scraper-go/internal/kwsync/kwsync.go index ee37787..3ec4417 100644 --- a/scraper-go/internal/kwsync/kwsync.go +++ b/scraper-go/internal/kwsync/kwsync.go @@ -38,6 +38,11 @@ func NewConsumer(rdb *redis.Client, kwStore *keywords.Store) *Consumer { // Start inicia o polling em background. // Cancela quando ctx for cancelado (shutdown do servidor). func (c *Consumer) Start(ctx context.Context) { + if !enabled() { + slog.Info("kwsync: consumer desabilitado", "env", "KWSYNC_ENABLED") + return + } + slog.Info("kwsync: consumer iniciado", "interval", pollInterval) go func() { @@ -58,6 +63,15 @@ func (c *Consumer) Start(ctx context.Context) { }() } +func enabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("KWSYNC_ENABLED"))) { + case "1", "true", "yes", "y", "on": + return true + default: + return false + } +} + // process drena a fila inteira e processa cada keyword. func (c *Consumer) process(ctx context.Context) error { raws, err := c.rdb.LRange(ctx, pendingKey, 0, -1).Result() diff --git a/scraper-go/internal/pipeline/cache_key.go b/scraper-go/internal/pipeline/cache_key.go index 43a1712..407654a 100644 --- a/scraper-go/internal/pipeline/cache_key.go +++ b/scraper-go/internal/pipeline/cache_key.go @@ -9,7 +9,7 @@ import ( ) func BuildCacheKey(config SearchConfig) string { - normalizedKeywords := keywords.NormalizeKeywords(config.Keywords) + normalizedKeywords := keywords.GenerateSearchKeywords(config.Keywords) sort.Strings(normalizedKeywords) sources := normalizeCacheValues(config.Sources) diff --git a/scraper-go/internal/pipeline/scrape.go b/scraper-go/internal/pipeline/scrape.go index 3e2f61b..0665fb7 100644 --- a/scraper-go/internal/pipeline/scrape.go +++ b/scraper-go/internal/pipeline/scrape.go @@ -27,7 +27,7 @@ type SearchConfig struct { } func normalizeSearchConfig(config SearchConfig) SearchConfig { - config.Keywords = keywords.NormalizeKeywords(config.Keywords) + config.Keywords = keywords.GenerateSearchKeywords(config.Keywords) return config }