mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2464 from github/robertbrignull/invocation-rate-limiter

Browse Source 
Move the invocation rate limiter class out of helpers.ts
This commit is contained in:
Robert
2023-05-31 16:45:17 +01:00
committed by GitHub
parent 157210ffc5 50c46b603a
commit bc342022cb
6 changed files with 243 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
extensions/ql-vscode/src/codeql-cli/distribution.ts
View File

@@ -6,12 +6,7 @@ import * as semver from "semver";
import { URL } from "url";
import { ExtensionContext, Event } from "vscode";
import { DistributionConfig } from "../config";
import {
InvocationRateLimiter,
InvocationRateLimiterResultKind,
showAndLogErrorMessage,
showAndLogWarningMessage,
} from "../helpers";
import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
import { extLogger } from "../common";
import { getCodeQlCliVersion } from "./cli-version";
import {
@@ -24,6 +19,10 @@ import {
extractZipArchive,
getRequiredAssetName,
} from "../pure/distribution";
import {
InvocationRateLimiter,
InvocationRateLimiterResultKind,
} from "../common/invocation-rate-limiter";
/**
* distribution.ts
@@ -76,7 +75,7 @@ export class DistributionManager implements DistributionProvider {
extensionContext,
);
this.updateCheckRateLimiter = new InvocationRateLimiter(
extensionContext,
extensionContext.globalState,
"extensionSpecificDistributionUpdateCheck",
() =>
this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),

89
extensions/ql-vscode/src/common/invocation-rate-limiter.ts Normal file
View File

@@ -0,0 +1,89 @@
import { Memento } from "./memento";
/**
* Provides a utility method to invoke a function only if a minimum time interval has elapsed since
* the last invocation of that function.
*/
export class InvocationRateLimiter<T> {
constructor(
private readonly globalState: Memento,
private readonly funcIdentifier: string,
private readonly func: () => Promise<T>,
private readonly createDate: (dateString?: string) => Date = (s) =>
s ? new Date(s) : new Date(),
) {}
/**
* Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
*/
public async invokeFunctionIfIntervalElapsed(
minSecondsSinceLastInvocation: number,
): Promise<InvocationRateLimiterResult<T>> {
const updateCheckStartDate = this.createDate();
const lastInvocationDate = this.getLastInvocationDate();
if (
minSecondsSinceLastInvocation &&
lastInvocationDate &&
lastInvocationDate <= updateCheckStartDate &&
lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
updateCheckStartDate.getTime()
) {
return createRateLimitedResult();
}
const result = await this.func();
await this.setLastInvocationDate(updateCheckStartDate);
return createInvokedResult(result);
}
private getLastInvocationDate(): Date | undefined {
const maybeDateString: string | undefined = this.globalState.get(
InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
);
return maybeDateString ? this.createDate(maybeDateString) : undefined;
}
private async setLastInvocationDate(date: Date): Promise<void> {
return await this.globalState.update(
InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
date,
);
}
private static readonly _invocationRateLimiterPrefix =
"invocationRateLimiter_lastInvocationDate_";
}
export enum InvocationRateLimiterResultKind {
Invoked,
RateLimited,
}
/**
* The function was invoked and returned the value `result`.
*/
interface InvokedResult<T> {
kind: InvocationRateLimiterResultKind.Invoked;
result: T;
}
/**
* The function was not invoked as the minimum interval since the last invocation had not elapsed.
*/
interface RateLimitedResult {
kind: InvocationRateLimiterResultKind.RateLimited;
}
type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
function createInvokedResult<T>(result: T): InvokedResult<T> {
return {
kind: InvocationRateLimiterResultKind.Invoked,
result,
};
}
function createRateLimitedResult(): RateLimitedResult {
return {
kind: InvocationRateLimiterResultKind.RateLimited,
};
}

109
extensions/ql-vscode/src/helpers.ts
View File

@@ -10,14 +10,7 @@ import { glob } from "glob";
import { load } from "js-yaml";
import { join, basename, dirname } from "path";
import { dirSync } from "tmp-promise";
import {
ExtensionContext,
Uri,
window as Window,
workspace,
env,
WorkspaceFolder,
} from "vscode";
import { Uri, window as Window, workspace, env, WorkspaceFolder } from "vscode";
import { CodeQLCliServer, QlpacksInfo } from "./codeql-cli/cli";
import { UserCancellationException } from "./common/vscode/progress";
import { extLogger, OutputChannelLogger } from "./common";
@@ -363,106 +356,6 @@ export async function prepareCodeTour(
}
}
/**
* Provides a utility method to invoke a function only if a minimum time interval has elapsed since
* the last invocation of that function.
*/
export class InvocationRateLimiter<T> {
constructor(
extensionContext: ExtensionContext,
funcIdentifier: string,
func: () => Promise<T>,
createDate: (dateString?: string) => Date = (s) =>
s ? new Date(s) : new Date(),
) {
this._createDate = createDate;
this._extensionContext = extensionContext;
this._func = func;
this._funcIdentifier = funcIdentifier;
}
/**
* Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
*/
public async invokeFunctionIfIntervalElapsed(
minSecondsSinceLastInvocation: number,
): Promise<InvocationRateLimiterResult<T>> {
const updateCheckStartDate = this._createDate();
const lastInvocationDate = this.getLastInvocationDate();
if (
minSecondsSinceLastInvocation &&
lastInvocationDate &&
lastInvocationDate <= updateCheckStartDate &&
lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
updateCheckStartDate.getTime()
) {
return createRateLimitedResult();
}
const result = await this._func();
await this.setLastInvocationDate(updateCheckStartDate);
return createInvokedResult(result);
}
private getLastInvocationDate(): Date | undefined {
const maybeDateString: string | undefined =
this._extensionContext.globalState.get(
InvocationRateLimiter._invocationRateLimiterPrefix +
this._funcIdentifier,
);
return maybeDateString ? this._createDate(maybeDateString) : undefined;
}
private async setLastInvocationDate(date: Date): Promise<void> {
return await this._extensionContext.globalState.update(
InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier,
date,
);
}
private readonly _createDate: (dateString?: string) => Date;
private readonly _extensionContext: ExtensionContext;
private readonly _func: () => Promise<T>;
private readonly _funcIdentifier: string;
private static readonly _invocationRateLimiterPrefix =
"invocationRateLimiter_lastInvocationDate_";
}
export enum InvocationRateLimiterResultKind {
Invoked,
RateLimited,
}
/**
* The function was invoked and returned the value `result`.
*/
interface InvokedResult<T> {
kind: InvocationRateLimiterResultKind.Invoked;
result: T;
}
/**
* The function was not invoked as the minimum interval since the last invocation had not elapsed.
*/
interface RateLimitedResult {
kind: InvocationRateLimiterResultKind.RateLimited;
}
type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
function createInvokedResult<T>(result: T): InvokedResult<T> {
return {
kind: InvocationRateLimiterResultKind.Invoked,
result,
};
}
function createRateLimitedResult(): RateLimitedResult {
return {
kind: InvocationRateLimiterResultKind.RateLimited,
};
}
export interface QlPacksForLanguage {
/** The name of the pack containing the dbscheme. */
dbschemePack: string;

143
extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts Normal file
View File

@@ -0,0 +1,143 @@
import type { Memento } from "vscode";
import { InvocationRateLimiter } from "../../../src/common/invocation-rate-limiter";
describe("Invocation rate limiter", () => {
// 1 January 2020
let currentUnixTime = 1577836800;
function createDate(dateString?: string): Date {
if (dateString) {
return new Date(dateString);
}
const numMillisecondsPerSecond = 1000;
return new Date(currentUnixTime * numMillisecondsPerSecond);
}
function createInvocationRateLimiter<T>(
funcIdentifier: string,
func: () => Promise<T>,
): InvocationRateLimiter<T> {
return new InvocationRateLimiter(
new MockMemento(),
funcIdentifier,
func,
(s) => createDate(s),
);
}
class MockMemento implements Memento {
keys(): readonly string[] {
throw new Error("Method not implemented.");
}
map = new Map<any, any>();
/**
* Return a value.
*
* @param key A string.
* @param defaultValue A value that should be returned when there is no
* value (`undefined`) with the given key.
* @return The stored value or the defaultValue.
*/
get<T>(key: string, defaultValue?: T): T {
return this.map.has(key) ? this.map.get(key) : defaultValue;
}
/**
* Store a value. The value must be JSON-stringifyable.
*
* @param key A string.
* @param value A value. MUST not contain cyclic references.
*/
async update(key: string, value: any): Promise<void> {
this.map.set(key, value);
}
}
it("initially invokes function", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncCalled).toBe(1);
});
it("doesn't invoke function again if no time has passed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncCalled).toBe(1);
});
it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
currentUnixTime += 1;
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
expect(numTimesFuncCalled).toBe(1);
});
it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
expect(numTimesFuncCalled).toBe(2);
});
it("invokes function again after requested time since last invocation has elapsed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
currentUnixTime += 1;
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
expect(numTimesFuncCalled).toBe(2);
});
it("invokes functions with different rate limiters", async () => {
let numTimesFuncACalled = 0;
const invocationRateLimiterA = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncACalled++;
},
);
let numTimesFuncBCalled = 0;
const invocationRateLimiterB = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncBCalled++;
},
);
await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncACalled).toBe(1);
expect(numTimesFuncBCalled).toBe(1);
});
});

6
extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
View File

@@ -286,7 +286,7 @@ describe("Launcher path", () => {
const manager = new DistributionManager(
{ customCodeQlPath: pathToCmd } as any,
{} as any,
undefined as any,
{} as any,
);
const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -304,7 +304,7 @@ describe("Launcher path", () => {
const manager = new DistributionManager(
{ customCodeQlPath: pathToCmd } as any,
{} as any,
undefined as any,
{} as any,
);
const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -319,7 +319,7 @@ describe("Launcher path", () => {
const manager = new DistributionManager(
{ customCodeQlPath: pathToCmd } as any,
{} as any,
undefined as any,
{} as any,
);
const result = await manager.getCodeQlPathWithoutVersionCheck();

238
extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts
View File

@@ -1,17 +1,4 @@
import {
EnvironmentVariableCollection,
EnvironmentVariableMutator,
Event,
ExtensionContext,
ExtensionMode,
Memento,
SecretStorage,
SecretStorageChangeEvent,
Uri,
window,
workspace,
WorkspaceFolder,
} from "vscode";
import { Uri, window, workspace, WorkspaceFolder } from "vscode";
import { dump } from "js-yaml";
import * as tmp from "tmp";
import { join } from "path";
@@ -28,7 +15,6 @@ import { DirResult } from "tmp";
import {
getFirstWorkspaceFolder,
getInitialQueryContents,
InvocationRateLimiter,
isFolderAlreadyInWorkspace,
isLikelyDatabaseRoot,
isLikelyDbLanguageFolder,
@@ -45,118 +31,6 @@ import { Setting } from "../../../src/config";
import { createMockCommandManager } from "../../__mocks__/commandsMock";
describe("helpers", () => {
describe("Invocation rate limiter", () => {
// 1 January 2020
let currentUnixTime = 1577836800;
function createDate(dateString?: string): Date {
if (dateString) {
return new Date(dateString);
}
const numMillisecondsPerSecond = 1000;
return new Date(currentUnixTime * numMillisecondsPerSecond);
}
function createInvocationRateLimiter<T>(
funcIdentifier: string,
func: () => Promise<T>,
): InvocationRateLimiter<T> {
return new InvocationRateLimiter(
new MockExtensionContext(),
funcIdentifier,
func,
(s) => createDate(s),
);
}
it("initially invokes function", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncCalled).toBe(1);
});
it("doesn't invoke function again if no time has passed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncCalled).toBe(1);
});
it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
currentUnixTime += 1;
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
expect(numTimesFuncCalled).toBe(1);
});
it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
expect(numTimesFuncCalled).toBe(2);
});
it("invokes function again after requested time since last invocation has elapsed", async () => {
let numTimesFuncCalled = 0;
const invocationRateLimiter = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncCalled++;
},
);
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
currentUnixTime += 1;
await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
expect(numTimesFuncCalled).toBe(2);
});
it("invokes functions with different rate limiters", async () => {
let numTimesFuncACalled = 0;
const invocationRateLimiterA = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncACalled++;
},
);
let numTimesFuncBCalled = 0;
const invocationRateLimiterB = createInvocationRateLimiter(
"funcid",
async () => {
numTimesFuncBCalled++;
},
);
await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
expect(numTimesFuncACalled).toBe(1);
expect(numTimesFuncBCalled).toBe(1);
});
});
describe("codeql-database.yml tests", () => {
let dir: tmp.DirResult;
let language: QueryLanguage;
@@ -250,116 +124,6 @@ describe("helpers", () => {
});
});
class MockExtensionContext implements ExtensionContext {
extensionMode: ExtensionMode = 3;
subscriptions: Array<{ dispose(): unknown }> = [];
workspaceState: Memento = new MockMemento();
globalState = new MockGlobalStorage();
extensionPath = "";
asAbsolutePath(_relativePath: string): string {
throw new Error("Method not implemented.");
}
storagePath = "";
globalStoragePath = "";
logPath = "";
extensionUri = Uri.parse("");
environmentVariableCollection = new MockEnvironmentVariableCollection();
secrets = new MockSecretStorage();
storageUri = Uri.parse("");
globalStorageUri = Uri.parse("");
logUri = Uri.parse("");
extension: any;
}
class MockEnvironmentVariableCollection
implements EnvironmentVariableCollection
{
[Symbol.iterator](): Iterator<
[variable: string, mutator: EnvironmentVariableMutator],
any,
undefined
> {
throw new Error("Method not implemented.");
}
persistent = false;
replace(_variable: string, _value: string): void {
throw new Error("Method not implemented.");
}
append(_variable: string, _value: string): void {
throw new Error("Method not implemented.");
}
prepend(_variable: string, _value: string): void {
throw new Error("Method not implemented.");
}
get(_variable: string): EnvironmentVariableMutator | undefined {
throw new Error("Method not implemented.");
}
forEach(
_callback: (
variable: string,
mutator: EnvironmentVariableMutator,
collection: EnvironmentVariableCollection,
) => any,
_thisArg?: any,
): void {
throw new Error("Method not implemented.");
}
delete(_variable: string): void {
throw new Error("Method not implemented.");
}
clear(): void {
throw new Error("Method not implemented.");
}
}
class MockMemento implements Memento {
keys(): readonly string[] {
throw new Error("Method not implemented.");
}
map = new Map<any, any>();
/**
* Return a value.
*
* @param key A string.
* @param defaultValue A value that should be returned when there is no
* value (`undefined`) with the given key.
* @return The stored value or the defaultValue.
*/
get<T>(key: string, defaultValue?: T): T {
return this.map.has(key) ? this.map.get(key) : defaultValue;
}
/**
* Store a value. The value must be JSON-stringifyable.
*
* @param key A string.
* @param value A value. MUST not contain cyclic references.
*/
async update(key: string, value: any): Promise<void> {
this.map.set(key, value);
}
}
class MockGlobalStorage extends MockMemento {
public setKeysForSync(_keys: string[]): void {
return;
}
}
class MockSecretStorage implements SecretStorage {
get(_key: string): Thenable<string | undefined> {
throw new Error("Method not implemented.");
}
store(_key: string, _value: string): Thenable<void> {
throw new Error("Method not implemented.");
}
delete(_key: string): Thenable<void> {
throw new Error("Method not implemented.");
}
onDidChange!: Event<SecretStorageChangeEvent>;
}
it("should report stream progress", () => {
const progressSpy = jest.fn();
const mockReadable = {