Add "pack install" and "pack download" commands

extensions/ql-vscode/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [UNRELEASED]
 
 - Fix a bug where the results view moved column even when it was already visible. [#1070](https://github.com/github/vscode-codeql/pull/1070)
+- Add packaging-related commands. _CodeQL: Download Packs_ downloads query packs from the package registry that can be run locally, and _CodeQL: Install Pack Dependencies_ installs dependencies for packs in your workspace. [#1076](https://github.com/github/vscode-codeql/pull/1076)
 
 ## 1.5.9 - 17 December 2021
 

extensions/ql-vscode/package.json

@@ -373,6 +373,14 @@
         "command": "codeQL.clearCache",
         "title": "CodeQL: Clear Cache"
       },
+      {
+        "command": "codeQL.installPackDependencies",
+        "title": "CodeQL: Install Pack Dependencies"
+      },
+      {
+        "command": "codeQL.downloadPacks",
+        "title": "CodeQL: Download Packs"
+      },
       {
         "command": "codeQLDatabases.setCurrentDatabase",
         "title": "Set Current Database"

extensions/ql-vscode/src/cli.ts

@@ -845,6 +845,14 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Downloads a specified pack.
+   * @param packs The `<package-scope/name[@version]>` of the packs to download.
+   */
+  async packDownload(packs: string[]) {
+    return this.runJsonCodeQlCliCommand(['pack', 'download'], packs, 'Downloading packs');
+  }
+
   async packInstall(dir: string) {
     return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
   }
@@ -1191,6 +1199,11 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_OLD_EVAL_STATS = new SemVer('2.7.4');
 
+  /**
+   * CLI version where packaging was introduced.
+   */
+  public static CLI_VERSION_WITH_PACKAGING = new SemVer('2.6.0');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1242,4 +1255,8 @@ export class CliVersionConstraint {
   async supportsOldEvalStats() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_OLD_EVAL_STATS);
   }
+
+  async supportsPackaging() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PACKAGING);
+  }
 }

extensions/ql-vscode/src/extension.ts

@@ -83,6 +83,7 @@ import { RemoteQuery } from './remote-queries/remote-query';
 import { URLSearchParams } from 'url';
 import { RemoteQueriesInterfaceManager } from './remote-queries/remote-queries-interface';
 import { sampleRemoteQuery, sampleRemoteQueryResult } from './remote-queries/sample-data';
+import { handleDownloadPacks, handleInstallPackDependencies } from './packaging';
 
 /**
  * extension.ts
@@ -922,6 +923,26 @@ async function activateWithInstalledDistribution(
       }
     }));
 
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.installPackDependencies', async (
+      progress: ProgressCallback
+    ) =>
+      await handleInstallPackDependencies(cliServer, progress),
+      {
+        title: 'Installing pack dependencies',
+      }
+    ));
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.downloadPacks', async (
+      progress: ProgressCallback
+    ) =>
+      await handleDownloadPacks(cliServer, progress),
+      {
+        title: 'Downloading packs',
+      }
+    ));
+
   commands.registerCommand('codeQL.showLogs', () => {
     logger.show();
   });

extensions/ql-vscode/src/packaging.ts

@@ -0,0 +1,142 @@
+import { CliVersionConstraint, CodeQLCliServer } from './cli';
+import {
+  getOnDiskWorkspaceFolders,
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+} from './helpers';
+import { QuickPickItem, window } from 'vscode';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
+import { logger } from './logging';
+
+const QUERY_PACKS = [
+  'codeql/cpp-queries',
+  'codeql/csharp-queries',
+  'codeql/go-queries',
+  'codeql/java-queries',
+  'codeql/javascript-queries',
+  'codeql/python-queries',
+  'codeql/ruby-queries',
+  'codeql/csharp-solorigate-queries',
+  'codeql/javascript-experimental-atm-queries',
+];
+
+/**
+ * Prompts user to choose packs to download, and downloads them.
+ *
+ * @param cliServer The CLI server.
+ * @param progress A progress callback.
+ */
+export async function handleDownloadPacks(
+  cliServer: CodeQLCliServer,
+  progress: ProgressCallback,
+): Promise<void> {
+  if (!(await cliServer.cliConstraints.supportsPackaging())) {
+    throw new Error(`Packaging commands are not supported by this version of CodeQL. Please upgrade to v${CliVersionConstraint.CLI_VERSION_WITH_PACKAGING
+      } or later.`);
+  }
+  progress({
+    message: 'Choose packs to download',
+    step: 1,
+    maxStep: 2,
+  });
+  let packsToDownload: string[] = [];
+  const queryPackOption = 'Download all core query packs';
+  const customPackOption = 'Download custom specified pack';
+  const quickpick = await window.showQuickPick(
+    [queryPackOption, customPackOption],
+    { ignoreFocusOut: true }
+  );
+  if (quickpick === queryPackOption) {
+    packsToDownload = QUERY_PACKS;
+  } else if (quickpick === customPackOption) {
+    const customPack = await window.showInputBox({
+      prompt:
+        'Enter the <package-scope/name[@version]> of the pack to download',
+      ignoreFocusOut: true,
+    });
+    if (customPack) {
+      packsToDownload.push(customPack);
+    } else {
+      throw new UserCancellationException('No pack specified.');
+    }
+  }
+  if (packsToDownload?.length > 0) {
+    progress({
+      message: 'Downloading packs. This may take a few minutes.',
+      step: 2,
+      maxStep: 2,
+    });
+    try {
+      await cliServer.packDownload(packsToDownload);
+      void showAndLogInformationMessage('Finished downloading packs.');
+    } catch (error) {
+      void showAndLogErrorMessage(
+        'Unable to download all packs. See log for more details.'
+      );
+    }
+  }
+}
+
+interface QLPackQuickPickItem extends QuickPickItem {
+  packRootDir: string[];
+}
+
+/**
+ * Prompts user to choose packs to install, and installs them.
+ *
+ * @param cliServer The CLI server.
+ * @param progress A progress callback.
+ */
+export async function handleInstallPackDependencies(
+  cliServer: CodeQLCliServer,
+  progress: ProgressCallback,
+): Promise<void> {
+  if (!(await cliServer.cliConstraints.supportsPackaging())) {
+    throw new Error(`Packaging commands are not supported by this version of CodeQL. Please upgrade to v${CliVersionConstraint.CLI_VERSION_WITH_PACKAGING
+      } or later.`);
+  }
+  progress({
+    message: 'Choose packs to install dependencies for',
+    step: 1,
+    maxStep: 2,
+  });
+  const workspacePacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
+  const quickPickItems = Object.entries(workspacePacks).map<QLPackQuickPickItem>(([key, value]) => ({
+    label: key,
+    packRootDir: value,
+  }));
+  const packsToInstall = await window.showQuickPick(quickPickItems, {
+    placeHolder: 'Select packs to install dependencies for',
+    canPickMany: true,
+    ignoreFocusOut: true,
+  });
+  if (packsToInstall && packsToInstall.length > 0) {
+    progress({
+      message: 'Installing dependencies. This may take a few minutes.',
+      step: 2,
+      maxStep: 2,
+    });
+    const failedPacks = [];
+    const errors = [];
+    for (const pack of packsToInstall) {
+      try {
+        for (const dir of pack.packRootDir) {
+          await cliServer.packInstall(dir);
+        }
+      } catch (error) {
+        failedPacks.push(pack.label);
+        errors.push(error);
+      }
+    }
+    if (failedPacks.length > 0) {
+      void logger.log(`Errors:\n${errors.join('\n')}`);
+      throw new Error(
+        `Unable to install pack dependencies for: ${failedPacks.join(', ')}. See log for more details.`
+      );
+    } else {
+      void showAndLogInformationMessage('Finished installing pack dependencies.');
+    }
+  } else {
+    throw new UserCancellationException('No packs selected.');
+  }
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-invalid-pack/qlpack.yml

@@ -0,0 +1,4 @@
+name: foo/bar
+version: 0.0.0
+dependencies:
+  foo/baz: '*'

extensions/ql-vscode/src/vscode-tests/cli-integration/packaging.test.ts

@@ -0,0 +1,128 @@
+import * as sinon from 'sinon';
+import { extensions, window } from 'vscode';
+import 'mocha';
+import * as path from 'path';
+
+import * as pq from 'proxyquire';
+
+import { CliVersionConstraint, CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+import { expect } from 'chai';
+
+const proxyquire = pq.noPreserveCache();
+
+describe('Packaging commands', function() {
+  let sandbox: sinon.SinonSandbox;
+
+  // up to 3 minutes per test
+  this.timeout(3 * 60 * 1000);
+
+  let cli: CodeQLCliServer;
+  let progress: sinon.SinonSpy;
+  let quickPickSpy: sinon.SinonStub;
+  let inputBoxSpy: sinon.SinonStub;
+  let showAndLogErrorMessageSpy: sinon.SinonStub;
+  let showAndLogInformationMessageSpy: sinon.SinonStub;
+  let mod: any;
+
+  beforeEach(async function() {
+    sandbox = sinon.createSandbox();
+
+    const extension = await extensions
+      .getExtension<CodeQLExtensionInterface | Record<string, never>>(
+        'GitHub.vscode-codeql'
+      )!
+      .activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error(
+        'Extension not initialized. Make sure cli is downloaded and installed properly.'
+      );
+    }
+    if (!(await cli.cliConstraints.supportsPackaging())) {
+      console.log(`Packaging commands are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_WITH_PACKAGING
+        }. Skipping this test.`);
+      this.skip();
+    }
+    progress = sandbox.spy();
+    quickPickSpy = sandbox.stub(window, 'showQuickPick');
+    inputBoxSpy = sandbox.stub(window, 'showInputBox');
+    showAndLogErrorMessageSpy = sandbox.stub();
+    showAndLogInformationMessageSpy = sandbox.stub();
+    mod = proxyquire('../../packaging', {
+      './helpers': {
+        showAndLogErrorMessage: showAndLogErrorMessageSpy,
+        showAndLogInformationMessage: showAndLogInformationMessageSpy,
+      },
+    });
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('should download all core query packs', async () => {
+    quickPickSpy.resolves('Download all core query packs');
+
+    await mod.handleDownloadPacks(cli, progress);
+    expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
+      'Finished downloading packs.'
+    );
+  });
+
+  it('should download valid user-specified pack', async () => {
+    quickPickSpy.resolves('Download custom specified pack');
+    inputBoxSpy.resolves('codeql/csharp-solorigate-queries');
+
+    await mod.handleDownloadPacks(cli, progress);
+    expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
+      'Finished downloading packs.'
+    );
+  });
+
+  it('should show error when downloading invalid user-specified pack', async () => {
+    quickPickSpy.resolves('Download custom specified pack');
+    inputBoxSpy.resolves('foo/not-a-real-pack@0.0.1');
+
+    await mod.handleDownloadPacks(cli, progress);
+
+    expect(showAndLogErrorMessageSpy.firstCall.args[0]).to.contain(
+      'Unable to download all packs.'
+    );
+  });
+
+  it('should install valid workspace pack', async () => {
+    const rootDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration/data');
+    quickPickSpy.resolves([
+      {
+        label: 'integration-test-queries-javascript',
+        packRootDir: [rootDir],
+      },
+    ]);
+
+    await mod.handleInstallPackDependencies(cli, progress);
+    expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
+      'Finished installing pack dependencies.'
+    );
+  });
+
+  it('should throw an error when installing invalid workspace pack', async () => {
+    const rootDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration/data-invalid-pack');
+    quickPickSpy.resolves([
+      {
+        label: 'foo/bar',
+        packRootDir: [rootDir],
+      },
+    ]);
+
+    try {
+      // expect this to throw an error
+      await mod.handleInstallPackDependencies(cli, progress);
+      // This line should not be reached
+      expect(true).to.be.false;
+    } catch (error) {
+      expect(error.message).to.contain('Unable to install pack dependencies');
+    }
+  });
+});