Skip to content

Commit 899efac

Browse files
Harshi-Shah-CSHarshi-Shah-CS
authored
Merge pull request #176 from contentstack/CL-4192
fix: paginate repositories and parse all remote URL formats
2 parents b5025af + daa6993 commit 899efac

5 files changed

Lines changed: 275 additions & 27 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@contentstack/cli-launch",
3-
"version": "1.10.1",
3+
"version": "1.11.0",
44
"description": "Launch related operations",
55
"author": "Contentstack CLI",
66
"bin": {

‎src/adapters/github.test.ts‎

Lines changed: 179 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { githubAdapter as cliUtilitiesJestMock } from '../test/mocks/cli-utilities';
2-
import GitHub from './github';
2+
import GitHub, { MAX_REPOSITORY_PAGES } from './github';
33
import { getRemoteUrls } from '../util/create-git-meta';
44
import { repositoriesQuery, userConnectionsQuery } from '../graphql';
55
import BaseClass from './base-class';
@@ -109,7 +109,15 @@ describe('GitHub Adapter', () => {
109109
});
110110

111111
describe('checkGitRemoteAvailableAndValid', () => {
112-
const repositoriesResponse = { data: { repositories } };
112+
const repositoriesResponse = {
113+
data: {
114+
repositories: {
115+
edges: repositories.map((node) => ({ node })),
116+
pageData: { page: 1 },
117+
pageInfo: { hasNextPage: false },
118+
},
119+
},
120+
};
113121

114122
it(`should successfully check if the git remote is available and valid
115123
when the github remote URL is HTTPS based`, async () => {
@@ -129,7 +137,37 @@ describe('GitHub Adapter', () => {
129137

130138
expect(existsSync).toHaveBeenCalledWith('/home/project1/.git');
131139
expect(getRemoteUrls as jest.Mock).toHaveBeenCalledWith('/home/project1/.git/config');
132-
expect(apolloClient.query).toHaveBeenCalledWith({ query: repositoriesQuery });
140+
expect(apolloClient.query).toHaveBeenCalledWith({
141+
query: repositoriesQuery,
142+
variables: { page: 1, first: 100 },
143+
});
144+
expect(githubAdapterInstance.config.repository).toEqual({
145+
__typename: 'GitRepository',
146+
id: '647250661',
147+
url: 'https://github.com/test-user/eleventy-sample',
148+
name: 'eleventy-sample',
149+
fullName: 'test-user/eleventy-sample',
150+
defaultBranch: 'main',
151+
});
152+
expect(result).toBe(true);
153+
});
154+
155+
it(`should successfully check if the git remote is available and valid
156+
when the github remote URL embeds userinfo (https://user@github.com/...)`, async () => {
157+
(existsSync as jest.Mock).mockReturnValueOnce(true);
158+
(getRemoteUrls as jest.Mock).mockResolvedValueOnce({
159+
origin: 'https://test-user@github.com/test-user/eleventy-sample.git',
160+
});
161+
const apolloClient = {
162+
query: jest.fn().mockResolvedValueOnce(repositoriesResponse),
163+
} as any;
164+
const githubAdapterInstance = new GitHub({
165+
config: { projectBasePath: '/home/project1' },
166+
apolloClient: apolloClient,
167+
} as any);
168+
169+
const result = await githubAdapterInstance.checkGitRemoteAvailableAndValid();
170+
133171
expect(githubAdapterInstance.config.repository).toEqual({
134172
__typename: 'GitRepository',
135173
id: '647250661',
@@ -159,7 +197,10 @@ describe('GitHub Adapter', () => {
159197

160198
expect(existsSync).toHaveBeenCalledWith('/home/project1/.git');
161199
expect(getRemoteUrls as jest.Mock).toHaveBeenCalledWith('/home/project1/.git/config');
162-
expect(apolloClient.query).toHaveBeenCalledWith({ query: repositoriesQuery });
200+
expect(apolloClient.query).toHaveBeenCalledWith({
201+
query: repositoriesQuery,
202+
variables: { page: 1, first: 100 },
203+
});
163204
expect(githubAdapterInstance.config.repository).toEqual({
164205
__typename: 'GitRepository',
165206
id: '647250661',
@@ -281,7 +322,10 @@ describe('GitHub Adapter', () => {
281322

282323
expect(existsSync).toHaveBeenCalledWith('/home/project1/.git');
283324
expect(getRemoteUrls as jest.Mock).toHaveBeenCalledWith('/home/project1/.git/config');
284-
expect(apolloClient.query).toHaveBeenCalledWith({ query: repositoriesQuery });
325+
expect(apolloClient.query).toHaveBeenCalledWith({
326+
query: repositoriesQuery,
327+
variables: { page: 1, first: 100 },
328+
});
285329
expect(logMock).toHaveBeenCalledWith(
286330
'Repository not added to the GitHub app. Please add it to the app’s repository access list and try again.',
287331
'error',
@@ -290,6 +334,136 @@ describe('GitHub Adapter', () => {
290334
expect(err).toEqual(new Error('1'));
291335
expect(githubAdapterInstance.config.repository).toBeUndefined();
292336
});
337+
338+
it('should log an error and exit if the remote URL format is unsupported', async () => {
339+
(existsSync as jest.Mock).mockReturnValueOnce(true);
340+
(getRemoteUrls as jest.Mock).mockResolvedValueOnce({
341+
origin: 'https://gitlab.com/test-user/some-repo.git',
342+
});
343+
const apolloClient = {
344+
query: jest.fn().mockResolvedValueOnce(repositoriesResponse),
345+
} as any;
346+
const githubAdapterInstance = new GitHub({
347+
config: { projectBasePath: '/home/project1' },
348+
log: logMock,
349+
exit: exitMock,
350+
apolloClient: apolloClient,
351+
} as any);
352+
let err;
353+
354+
try {
355+
await githubAdapterInstance.checkGitRemoteAvailableAndValid();
356+
} catch (error: any) {
357+
err = error;
358+
}
359+
360+
expect(logMock).toHaveBeenCalledWith(
361+
'Unsupported Git remote URL format: https://gitlab.com/test-user/some-repo.git. Please use a standard GitHub HTTPS or SSH remote URL.',
362+
'error',
363+
);
364+
expect(exitMock).toHaveBeenCalledWith(1);
365+
expect(err).toEqual(new Error('1'));
366+
expect(githubAdapterInstance.config.repository).toBeUndefined();
367+
});
368+
369+
it('should paginate beyond the first page to find a repository on a later page', async () => {
370+
(existsSync as jest.Mock).mockReturnValueOnce(true);
371+
(getRemoteUrls as jest.Mock).mockResolvedValueOnce({
372+
origin: 'https://github.com/test-user/repo-301.git',
373+
});
374+
375+
const PER_PAGE = 100;
376+
const FULL_PAGES = 3; // target sits on the 4th (partial) page, well within the cap
377+
const targetRepo = {
378+
__typename: 'GitRepository',
379+
id: '301',
380+
url: 'https://github.com/test-user/repo-301',
381+
name: 'repo-301',
382+
fullName: 'test-user/repo-301',
383+
defaultBranch: 'main',
384+
};
385+
386+
const apolloClient = {
387+
query: jest.fn().mockImplementation(({ variables }) => {
388+
const { page } = variables;
389+
const edges =
390+
page <= FULL_PAGES
391+
? Array.from({ length: PER_PAGE }, (_, i) => ({
392+
node: {
393+
__typename: 'GitRepository',
394+
id: `${(page - 1) * PER_PAGE + i}`,
395+
url: `https://github.com/test-user/repo-${(page - 1) * PER_PAGE + i}`,
396+
name: `repo-${(page - 1) * PER_PAGE + i}`,
397+
fullName: `test-user/repo-${(page - 1) * PER_PAGE + i}`,
398+
defaultBranch: 'main',
399+
},
400+
}))
401+
: [{ node: targetRepo }];
402+
return Promise.resolve({ data: { repositories: { edges, pageData: { page }, pageInfo: { hasNextPage: false } } } });
403+
}),
404+
} as any;
405+
const githubAdapterInstance = new GitHub({
406+
config: { projectBasePath: '/home/project1' },
407+
apolloClient: apolloClient,
408+
} as any);
409+
410+
const result = await githubAdapterInstance.checkGitRemoteAvailableAndValid();
411+
412+
// 3 full pages + 1 partial page = 4 requests.
413+
expect(apolloClient.query).toHaveBeenCalledTimes(FULL_PAGES + 1);
414+
expect(githubAdapterInstance.config.repository).toEqual(targetRepo);
415+
expect(result).toBe(true);
416+
});
417+
418+
it(`should cap pagination at ${MAX_REPOSITORY_PAGES} pages (1000 repositories) for consistency with the management service`, async () => {
419+
(existsSync as jest.Mock).mockReturnValueOnce(true);
420+
(getRemoteUrls as jest.Mock).mockResolvedValueOnce({
421+
origin: 'https://github.com/test-user/missing-repo.git',
422+
});
423+
424+
// Every page is full, so without a cap the loop would never stop on its own.
425+
const apolloClient = {
426+
query: jest.fn().mockImplementation(({ variables }) => {
427+
const { page, first } = variables;
428+
const edges = Array.from({ length: first }, (_, i) => ({
429+
node: {
430+
__typename: 'GitRepository',
431+
id: `${(page - 1) * first + i}`,
432+
url: `https://github.com/test-user/repo-${(page - 1) * first + i}`,
433+
name: `repo-${(page - 1) * first + i}`,
434+
fullName: `test-user/repo-${(page - 1) * first + i}`,
435+
defaultBranch: 'main',
436+
},
437+
}));
438+
return Promise.resolve({ data: { repositories: { edges, pageData: { page }, pageInfo: { hasNextPage: true } } } });
439+
}),
440+
} as any;
441+
const githubAdapterInstance = new GitHub({
442+
config: { projectBasePath: '/home/project1' },
443+
log: logMock,
444+
exit: exitMock,
445+
apolloClient: apolloClient,
446+
} as any);
447+
let err;
448+
449+
try {
450+
await githubAdapterInstance.checkGitRemoteAvailableAndValid();
451+
} catch (error: any) {
452+
err = error;
453+
}
454+
455+
expect(apolloClient.query).toHaveBeenCalledTimes(MAX_REPOSITORY_PAGES);
456+
expect(logMock).toHaveBeenCalledWith(
457+
expect.stringContaining('beyond the first 1000 repositories the GitHub App can access'),
458+
'error',
459+
);
460+
expect(logMock).not.toHaveBeenCalledWith(
461+
'Repository not added to the GitHub app. Please add it to the app’s repository access list and try again.',
462+
'error',
463+
);
464+
expect(exitMock).toHaveBeenCalledWith(1);
465+
expect(err).toEqual(new Error('1'));
466+
});
293467
});
294468

295469
describe('runGitHubFlow', () => {

‎src/adapters/github.ts‎

Lines changed: 66 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@ import { print } from '../util';
1111
import BaseClass from './base-class';
1212
import { getRemoteUrls } from '../util/create-git-meta';
1313
import { repositoriesQuery, userConnectionsQuery, importProjectMutation } from '../graphql';
14-
import { DeploymentStatus } from '../types';
14+
import { DeploymentStatus, Repository } from '../types';
1515
import { existsSync } from 'fs';
1616

17+
export const MAX_REPOSITORY_PAGES = 10;
18+
export const REPOSITORY_PAGE_SIZE = 100;
19+
1720
export default class GitHub extends BaseClass {
1821
/**
1922
* @method run - initialization function
@@ -71,10 +74,12 @@ export default class GitHub extends BaseClass {
7174
private async handleNewProject(): Promise<void> {
7275
// NOTE Step 1: Check is Github connected
7376
if (await this.checkGitHubConnected()) {
74-
// NOTE Step 2: check is the git remote available in the user's repo list
77+
// NOTE Step 2: Select org first; the GitRepositories query is org-scoped (guarded).
78+
await this.selectOrg();
79+
// NOTE Step 3: check is the git remote available in the user's repo list
7580
if (await this.checkGitRemoteAvailableAndValid()) {
7681
if (await this.checkUserGitHubAccess()) {
77-
// NOTE Step 3: check is the user has proper git access
82+
// NOTE Step 4: check is the user has proper git access
7883
await this.prepareForNewProjectCreation();
7984
}
8085
}
@@ -168,7 +173,6 @@ export default class GitHub extends BaseClass {
168173
const { token, apiKey } = configHandler.get(`tokens.${alias}`) ?? {};
169174
this.config.selectedStack = apiKey;
170175
this.config.deliveryToken = token;
171-
await this.selectOrg();
172176
print([
173177
{ message: '?', color: 'green' },
174178
{ message: 'Repository', bold: true },
@@ -292,16 +296,16 @@ export default class GitHub extends BaseClass {
292296
private extractRepoFullNameFromGithubRemoteURL(url: string) {
293297
let match;
294298

295-
// HTTPS format: https://github.com/owner/repo.git
296-
match = url.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)(\.git)?$/);
299+
// HTTPS format: https://[user[:token]@]github.com/owner/repo(.git)(/)
300+
match = url.match(/^https:\/\/(?:[^@/]+@)?github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/);
297301
if (match) {
298-
return `${match[1]}/${match[2].replace(/\.git$/, '')}`;
302+
return `${match[1]}/${match[2]}`;
299303
}
300304

301-
// SSH format: git@github.com:owner/repo.git
302-
match = url.match(/^git@github\.com:([^/]+)\/([^/]+)(\.git)?$/);
305+
// SSH format: git@github.com:owner/repo(.git) or ssh://git@github.com/owner/repo(.git)
306+
match = url.match(/^(?:ssh:\/\/)?git@github\.com[:/]([^/]+)\/([^/]+?)(?:\.git)?\/?$/);
303307
if (match) {
304-
return `${match[1]}/${match[2].replace(/\.git$/, '')}`;
308+
return `${match[1]}/${match[2]}`;
305309
}
306310
}
307311

@@ -331,10 +335,9 @@ export default class GitHub extends BaseClass {
331335
this.exit(1);
332336
}
333337

334-
let repositories;
338+
let repositories: Repository[] = [];
335339
try {
336-
const repositoriesQueryResponse = await this.apolloClient.query({ query: repositoriesQuery });
337-
repositories = repositoriesQueryResponse.data.repositories;
340+
repositories = await this.queryRepositories({ page: 1, first: REPOSITORY_PAGE_SIZE });
338341
} catch {
339342
this.log('GitHub app uninstalled. Please reconnect the app and try again', 'error');
340343
await this.connectToAdapterOnUi();
@@ -343,11 +346,30 @@ export default class GitHub extends BaseClass {
343346

344347
const repoFullName = this.extractRepoFullNameFromGithubRemoteURL(localRemoteUrl);
345348

349+
if (!repoFullName) {
350+
this.log(
351+
`Unsupported Git remote URL format: ${localRemoteUrl}. Please use a standard GitHub HTTPS or SSH remote URL.`,
352+
'error',
353+
);
354+
this.exit(1);
355+
}
356+
346357
this.config.repository = find(repositories, {
347358
fullName: repoFullName,
348359
});
349360

350361
if (!this.config.repository) {
362+
const checkedCount = MAX_REPOSITORY_PAGES * REPOSITORY_PAGE_SIZE;
363+
if (repositories.length >= checkedCount) {
364+
this.log(
365+
`"${repoFullName}" is beyond the first ${checkedCount} repositories the GitHub App can access. ` +
366+
'In your GitHub App\'s installation settings, under "Repository access", select ' +
367+
'"Only select repositories" and add the repository you want to deploy. Then re-run the command.',
368+
'error',
369+
);
370+
this.exit(1);
371+
}
372+
351373
this.log(
352374
'Repository not added to the GitHub app. Please add it to the app’s repository access list and try again.',
353375
'error',
@@ -358,6 +380,37 @@ export default class GitHub extends BaseClass {
358380
return true;
359381
}
360382

383+
/**
384+
* @method queryRepositories - Recursively fetch each page of repositories
385+
* accessible to the GitHub app, up to MAX_REPOSITORY_PAGES.
386+
*
387+
* @param {Record<string, any>} variables
388+
* @param {any[]} [repositoriesRes=[]]
389+
* @return {*} {Promise<any[]>}
390+
* @memberof GitHub
391+
*/
392+
async queryRepositories(
393+
variables: Record<string, any> = {},
394+
repositoriesRes: Repository[] = [],
395+
): Promise<Repository[]> {
396+
const first = typeof variables.first === 'number' ? variables.first : REPOSITORY_PAGE_SIZE;
397+
const page = typeof variables.page === 'number' ? variables.page : 1;
398+
399+
const { data: { repositories } } = await this.apolloClient.query({
400+
query: repositoriesQuery,
401+
variables: { ...variables, first, page },
402+
});
403+
404+
const edges = repositories?.edges ?? [];
405+
repositoriesRes.push(...map(edges, 'node'));
406+
407+
if (edges.length === first && page < MAX_REPOSITORY_PAGES) {
408+
return this.queryRepositories({ ...variables, first, page: page + 1 }, repositoriesRes);
409+
}
410+
411+
return repositoriesRes;
412+
}
413+
361414
/**
362415
* @method checkUserGitHubAccess - GitHub user access validation
363416
*

0 commit comments

Comments
 (0)