Skip to content

Commit 8b2c8ee

Browse files
cabljacCorieW
andauthored
feat: gen-schema-view updates (#2326)
* fix(gen-schema-view): run npm audit fix * refactor(gen-schema-view): extract config parsing to its own module * feat(gen-schema-views): add gemini gen schema Co-authored-by: Corie Watson <watson.corie@gmail.com> * chore(gen-schema-views): bump version * chore(gen-schema-views): add license headers * docs(gen-schema-views): update gemini docs * refactor(gen-schema-view): allow specifying of schema file name * fix(gen-schema-view): fix schema directory option in non-interactive, and update docs * docs(gen-schema-views): update guide --------- Co-authored-by: Corie Watson <watson.corie@gmail.com>
1 parent 2e62c46 commit 8b2c8ee

24 files changed

+5467
-2437
lines changed

firestore-bigquery-export/functions/package-lock.json

Lines changed: 27 additions & 102 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md

Lines changed: 175 additions & 321 deletions
Large diffs are not rendered by default.

firestore-bigquery-export/scripts/gen-schema-view/package-lock.json

Lines changed: 2180 additions & 1484 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

firestore-bigquery-export/scripts/gen-schema-view/package.json

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@firebaseextensions/fs-bq-schema-views",
3-
"version": "0.4.9",
3+
"version": "0.4.10",
44
"description": "Generate strongly-typed BigQuery Views based on raw JSON",
55
"main": "./lib/index.js",
66
"repository": {
@@ -31,13 +31,15 @@
3131
"author": "Jan Wyszynski <wyszynski@google.com>",
3232
"license": "Apache-2.0",
3333
"dependencies": {
34-
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36",
34+
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.38",
35+
"@genkit-ai/googleai": "^1.1.0",
3536
"@google-cloud/bigquery": "^6.0.3",
3637
"commander": "5.0.0",
3738
"firebase-admin": "^12.1.0",
3839
"firebase-functions": "^4.2.0",
3940
"fs-find": "^0.4.0",
4041
"generate-schema": "^2.6.0",
42+
"genkit": "^1.1.0",
4143
"glob": "7.1.5",
4244
"inquirer": "^6.4.0",
4345
"sql-formatter": "^2.3.3"
@@ -46,15 +48,16 @@
4648
"@types/chai": "^4.1.6",
4749
"@types/express": "^4.17.14",
4850
"@types/express-serve-static-core": "4.17.30",
51+
"@types/inquirer": "^9.0.7",
52+
"@types/jest": "29.5.0",
4953
"chai": "^4.2.0",
5054
"exec": "^0.2.1",
51-
"nyc": "^14.0.0",
52-
"rimraf": "^2.6.3",
53-
"ts-node": "^7.0.1",
54-
"typescript": "^4.9.3",
55-
"@types/jest": "29.5.0",
5655
"jest": "29.5.0",
5756
"mocked-env": "^1.3.2",
58-
"ts-jest": "29.1.2"
57+
"nyc": "^17.1.0",
58+
"rimraf": "^2.6.3",
59+
"ts-jest": "29.1.2",
60+
"ts-node": "^7.0.1",
61+
"typescript": "^4.9.3"
5962
}
6063
}
Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
import { parseConfig } from "../../../src/config";
2+
import { promptInquirer } from "../../../src/config/interactive";
3+
import {
4+
parseProgram,
5+
validateNonInteractiveParams,
6+
} from "../../../src/config/non-interactive";
7+
import { readSchemas } from "../../../src/schema-loader-utils";
8+
9+
// Mock dependencies
10+
jest.mock("../../../src/config/interactive", () => ({
11+
promptInquirer: jest.fn(),
12+
}));
13+
14+
jest.mock("../../../src/config/non-interactive", () => ({
15+
parseProgram: jest.fn(),
16+
validateNonInteractiveParams: jest.fn(),
17+
}));
18+
19+
jest.mock("../../../src/schema-loader-utils", () => ({
20+
readSchemas: jest.fn(),
21+
}));
22+
23+
// Mock process.exit to prevent tests from actually exiting
24+
const mockExit = jest.spyOn(process, "exit").mockImplementation((code) => {
25+
throw new Error(`Process exited with code ${code}`);
26+
});
27+
28+
describe("parseConfig", () => {
29+
beforeEach(() => {
30+
jest.clearAllMocks();
31+
});
32+
33+
describe("Non-interactive mode", () => {
34+
it("should return CLI config from command line arguments", async () => {
35+
// Setup mocks for non-interactive mode
36+
const mockProgram = {
37+
nonInteractive: true,
38+
project: "test-project",
39+
bigQueryProject: "test-bq-project",
40+
dataset: "test-dataset",
41+
tableNamePrefix: "test-prefix",
42+
schemaFiles: ["schema1.json", "schema2.json"],
43+
outputHelp: jest.fn(),
44+
};
45+
46+
const mockSchemas = {
47+
schema1: { fields: { field1: { type: "string" } } },
48+
schema2: { fields: { field2: { type: "number" } } },
49+
};
50+
51+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
52+
(validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
53+
(readSchemas as jest.Mock).mockReturnValue(mockSchemas);
54+
55+
const result = await parseConfig();
56+
57+
expect(parseProgram).toHaveBeenCalled();
58+
expect(validateNonInteractiveParams).toHaveBeenCalledWith(mockProgram);
59+
expect(readSchemas).toHaveBeenCalledWith(mockProgram.schemaFiles);
60+
expect(result).toEqual({
61+
agentSampleSize: 100,
62+
projectId: "test-project",
63+
bigQueryProjectId: "test-bq-project",
64+
datasetId: "test-dataset",
65+
tableNamePrefix: "test-prefix",
66+
schemas: mockSchemas,
67+
geminiAnalyzeCollectionPath: undefined,
68+
googleAiKey: undefined,
69+
schemaDirectory: undefined,
70+
useGemini: false,
71+
});
72+
});
73+
74+
it("should use project as bigQueryProject if not specified", async () => {
75+
// Setup mocks with missing bigQueryProject
76+
const mockProgram = {
77+
nonInteractive: true,
78+
project: "test-project",
79+
bigQueryProject: undefined,
80+
dataset: "test-dataset",
81+
tableNamePrefix: "test-prefix",
82+
schemaFiles: ["schema.json"],
83+
outputHelp: jest.fn(),
84+
};
85+
86+
const mockSchemas = { schema: { fields: { field: { type: "string" } } } };
87+
88+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
89+
(validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
90+
(readSchemas as jest.Mock).mockReturnValue(mockSchemas);
91+
92+
const result = await parseConfig();
93+
94+
expect(result.bigQueryProjectId).toBe("test-project");
95+
});
96+
97+
it("should use gemini if specified", async () => {
98+
// Setup mocks with useGemini = true
99+
const mockProgram = {
100+
nonInteractive: true,
101+
project: "test-project",
102+
bigQueryProject: "test-bq-project",
103+
dataset: "test-dataset",
104+
tableNamePrefix: "test-prefix",
105+
schemaFiles: ["schema.json"],
106+
useGemini: "test-collection",
107+
googleAiKey: "test-key",
108+
geminiAnalyzeCollectionPath: "test-collection",
109+
schemaDirectory: "test-directory",
110+
outputHelp: jest.fn(),
111+
};
112+
113+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
114+
(validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
115+
116+
const result = await parseConfig();
117+
118+
expect(result.useGemini).toBe(true);
119+
expect(result.googleAiKey).toBe("test-key");
120+
expect(result.geminiAnalyzeCollectionPath).toBe("test-collection");
121+
expect(result.schemaDirectory).toBe("test-directory");
122+
expect(result.agentSampleSize).toBe(100);
123+
});
124+
125+
it("should exit if required parameters are missing", async () => {
126+
const mockProgram = {
127+
nonInteractive: true,
128+
outputHelp: jest.fn(),
129+
};
130+
131+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
132+
(validateNonInteractiveParams as jest.Mock).mockReturnValue(false);
133+
134+
await expect(parseConfig()).rejects.toThrow("Process exited with code 1");
135+
expect(mockProgram.outputHelp).toHaveBeenCalled();
136+
expect(mockExit).toHaveBeenCalledWith(1);
137+
});
138+
});
139+
140+
describe("Interactive mode without Gemini", () => {
141+
it("should return CLI config from inquirer prompts", async () => {
142+
// Setup mocks for interactive mode
143+
const mockProgram = {
144+
nonInteractive: false,
145+
};
146+
147+
const mockPromptResponse = {
148+
projectId: "interactive-project",
149+
bigQueryProjectId: "interactive-bq-project",
150+
datasetId: "interactive-dataset",
151+
tableNamePrefix: "interactive-prefix",
152+
useGemini: false,
153+
schemaFiles: "schema1.json, schema2.json",
154+
};
155+
156+
const mockSchemas = {
157+
schema1: { fields: { field1: { type: "string" } } },
158+
schema2: { fields: { field2: { type: "number" } } },
159+
};
160+
161+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
162+
(promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
163+
(readSchemas as jest.Mock).mockReturnValue(mockSchemas);
164+
165+
const result = await parseConfig();
166+
167+
expect(parseProgram).toHaveBeenCalled();
168+
expect(promptInquirer).toHaveBeenCalled();
169+
// Expect the schemaFiles string to be passed as-is in an array
170+
expect(readSchemas).toHaveBeenCalledWith(["schema1.json, schema2.json"]);
171+
expect(result).toEqual({
172+
agentSampleSize: 100,
173+
projectId: "interactive-project",
174+
bigQueryProjectId: "interactive-bq-project",
175+
datasetId: "interactive-dataset",
176+
tableNamePrefix: "interactive-prefix",
177+
schemas: mockSchemas,
178+
geminiAnalyzeCollectionPath: undefined,
179+
googleAiKey: undefined,
180+
schemaDirectory: undefined,
181+
useGemini: false,
182+
});
183+
});
184+
185+
it("should properly handle schema file paths without trimming or splitting", async () => {
186+
const mockProgram = {
187+
nonInteractive: false,
188+
};
189+
190+
const mockPromptResponse = {
191+
project: "test-project",
192+
bigQueryProject: "test-bq-project",
193+
dataset: "test-dataset",
194+
tableNamePrefix: "test-prefix",
195+
useGemini: false,
196+
schemaFiles: " schema1.json, schema2.json , schema3.json",
197+
};
198+
199+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
200+
(promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
201+
(readSchemas as jest.Mock).mockReturnValue({});
202+
203+
await parseConfig();
204+
205+
// Verify that the schemaFiles string is passed as-is within an array
206+
expect(readSchemas).toHaveBeenCalledWith([
207+
" schema1.json, schema2.json , schema3.json",
208+
]);
209+
});
210+
});
211+
212+
describe("Interactive mode with Gemini", () => {
213+
it("should return CLI config from inquirer prompts", async () => {
214+
// Setup mocks for interactive mode
215+
const mockProgram = {
216+
nonInteractive: false,
217+
};
218+
219+
const mockPromptResponse = {
220+
projectId: "interactive-project",
221+
bigQueryProjectId: "interactive-bq-project",
222+
datasetId: "interactive-dataset",
223+
tableNamePrefix: "interactive-prefix",
224+
useGemini: true,
225+
googleAiKey: "test-key",
226+
geminiAnalyzeCollectionPath: "test-collection",
227+
schemaDirectory: "test-directory",
228+
};
229+
230+
// Although we set up mockSchemas, in Gemini mode readSchemas is not called.
231+
const mockSchemas = {
232+
schema1: { fields: { field1: { type: "string" } } },
233+
schema2: { fields: { field2: { type: "number" } } },
234+
};
235+
236+
(parseProgram as jest.Mock).mockReturnValue(mockProgram);
237+
(promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
238+
(readSchemas as jest.Mock).mockReturnValue(mockSchemas);
239+
240+
const result = await parseConfig();
241+
242+
expect(parseProgram).toHaveBeenCalled();
243+
expect(promptInquirer).toHaveBeenCalled();
244+
// In Gemini mode, schemaFiles may not be provided so readSchemas should not be called
245+
expect(readSchemas).not.toHaveBeenCalled();
246+
// Expect schemas to be an empty object in Gemini mode
247+
expect(result).toEqual({
248+
agentSampleSize: 100,
249+
projectId: "interactive-project",
250+
bigQueryProjectId: "interactive-bq-project",
251+
datasetId: "interactive-dataset",
252+
tableNamePrefix: "interactive-prefix",
253+
schemas: {},
254+
geminiAnalyzeCollectionPath: "test-collection",
255+
googleAiKey: "test-key",
256+
schemaDirectory: "test-directory",
257+
useGemini: true,
258+
});
259+
});
260+
});
261+
});

0 commit comments

Comments
 (0)