From d45dfdaa718506a871ba8f3ef6c923b1cbfdf5f9 Mon Sep 17 00:00:00 2001 From: Henning Dieterichs Date: Fri, 24 Mar 2023 14:06:22 +0100 Subject: [PATCH] Data extraction API improvements. --- .../src/js/api/DataExtractorApi.ts | 15 +- .../src/js/api/DataExtractorApiImpl.ts | 135 ++++++++++++++---- .../StringRangeExtractor.ts | 6 +- data-extraction/src/js/helpers/find.ts | 92 ++++++++++++ .../JsVisualizationSupport.ts | 4 +- 5 files changed, 220 insertions(+), 32 deletions(-) diff --git a/data-extraction/src/js/api/DataExtractorApi.ts b/data-extraction/src/js/api/DataExtractorApi.ts index cfb6ce7..aa69e24 100644 --- a/data-extraction/src/js/api/DataExtractorApi.ts +++ b/data-extraction/src/js/api/DataExtractorApi.ts @@ -53,6 +53,11 @@ export interface DataExtractor { * Must be unique among all data extractors. */ id: string; + + /** + * Filters the data to be extracted. + */ + dataCtor?: string; getExtractions( data: unknown, extractionCollector: ExtractionCollector, @@ -73,7 +78,11 @@ export interface DataExtractorContext { */ evalFn: (expression: string) => TEval; + expression: string | undefined; + variablesInScope: Record unknown>; + + extract(value: unknown): VisualizationData | undefined; } export interface DataExtraction { @@ -86,10 +95,12 @@ export interface DataExtraction { * A unique id identifying this extraction among all extractions. * Required to express extraction preferences. */ - id: string; + id?: string; + /** * A user friendly name of this extraction. */ - name: string; + name?: string; + extractData(): VisualizationData; } diff --git a/data-extraction/src/js/api/DataExtractorApiImpl.ts b/data-extraction/src/js/api/DataExtractorApiImpl.ts index 0211d39..0d486f8 100644 --- a/data-extraction/src/js/api/DataExtractorApiImpl.ts +++ b/data-extraction/src/js/api/DataExtractorApiImpl.ts @@ -7,7 +7,10 @@ import { ExtractionCollector, DataExtractorContext, } from "./DataExtractorApi"; -import { DataExtractorInfo } from "../../DataExtractionResult"; +import { + DataExtractorInfo, + VisualizationData, +} from "../../DataExtractionResult"; import { registerDefaultExtractors } from "./default-extractors"; import { LoadDataExtractorsFn } from "./LoadDataExtractorsFn"; import * as helpers from "../helpers"; @@ -41,36 +44,56 @@ export class DataExtractorApiImpl implements DataExtractorApi { preferredDataExtractorId: string | undefined, variablesInScope: Record unknown> ): JSONString { - const extractions = new Array(); - const extractionCollector: ExtractionCollector = { - addExtraction(extraction) { - extractions.push(extraction); - }, - }; + class ContextImpl implements DataExtractorContext { + constructor( + public readonly variablesInScope: Record unknown>, + public readonly expression: string | undefined, + public readonly evalFn: (expression: string) => T, + private readonly _api: DataExtractorApiImpl, + private readonly _parent: ContextImpl | undefined + ) {} - const context: DataExtractorContext = { - evalFn, + get _level(): number { + return this._parent ? this._parent._level + 1 : 0; + } + + extract(value: any): VisualizationData | undefined { + if (this._level > 10) { + throw new Error( + "extract() called too many times recursively" + ); + } + + const extractions = this._api.getExtractions( + value, + new ContextImpl( + this.variablesInScope, + undefined, + this.evalFn, + this._api, + this + ) + ); + if (extractions.length === 0) { + return undefined; + } + return extractions[0].extractData(); + } + } + + const rootContext = new ContextImpl( variablesInScope, - }; + removeEnd(removeStart(valueFn.toString(), "() => ("), ")").trim(), + evalFn, + this, + undefined + ); - DataExtractorApiImpl.lastContext = context; + DataExtractorApiImpl.lastContext = rootContext; const value = valueFn(); - - const extractors = new Array(); - - for (const fn of this.extractorSources.values()) { - fn((extractor) => { - extractors.push(extractor); - }, helpers); - } - - for (const e of [...this.extractors.values(), ...extractors]) { - e.getExtractions(value, extractionCollector, context); - } - + const extractions = this.getExtractions(value, rootContext); DataExtractorApiImpl.lastContext = undefined; - extractions.sort((a, b) => b.priority - a.priority); let usedExtraction = extractions[0]; if (!usedExtraction) { return this.toJson({ kind: "NoExtractors" } as DataResult); @@ -87,8 +110,8 @@ export class DataExtractorApiImpl implements DataExtractorApi { function mapExtractor(e: DataExtraction): DataExtractorInfo { return { - id: e.id as any, - name: e.name, + id: e.id! as any, + name: e.name!, priority: e.priority, }; } @@ -104,6 +127,50 @@ export class DataExtractorApiImpl implements DataExtractorApi { } as DataResult); } + public getExtractions( + value: unknown, + context: DataExtractorContext + ): DataExtraction[] { + const extractions = new Array(); + const extractors = new Array(); + + for (const fn of this.extractorSources.values()) { + fn((extractor) => { + extractors.push(extractor); + }, helpers); + } + + for (const e of [...this.extractors.values(), ...extractors]) { + if (e.dataCtor !== undefined) { + if ( + typeof value !== "object" || + value === null || + value.constructor.name !== e.dataCtor + ) { + continue; + } + } + e.getExtractions( + value, + { + addExtraction(extraction) { + if (extraction.id === undefined) { + extraction.id = e.id; + } + if (extraction.name === undefined) { + extraction.name = e.id; + } + extractions.push(extraction); + }, + }, + context + ); + } + extractions.sort((a, b) => b.priority - a.priority); + + return extractions; + } + public registerDefaultExtractors(preferExisting: boolean = false): void { // TODO consider preferExisting registerDefaultExtractors(this); @@ -120,3 +187,17 @@ export class DataExtractorApiImpl implements DataExtractorApi { } } } + +function removeStart(str: string, start: string): string { + if (str.startsWith(start)) { + return str.substr(start.length); + } + return str; +} + +function removeEnd(str: string, end: string): string { + if (str.endsWith(end)) { + return str.substr(0, str.length - end.length); + } + return str; +} diff --git a/data-extraction/src/js/api/default-extractors/StringRangeExtractor.ts b/data-extraction/src/js/api/default-extractors/StringRangeExtractor.ts index b79e469..45d44ba 100644 --- a/data-extraction/src/js/api/default-extractors/StringRangeExtractor.ts +++ b/data-extraction/src/js/api/default-extractors/StringRangeExtractor.ts @@ -50,7 +50,11 @@ export class StringRangeExtractor implements DataExtractor { for (let item of (data as (number | [number, number])[]).slice(1)) { if (typeof item === "string") { - item = context.evalFn(item); + try { + item = context.evalFn(item); + } catch (e) { + return; + } if (item === undefined) { continue; } diff --git a/data-extraction/src/js/helpers/find.ts b/data-extraction/src/js/helpers/find.ts index 6ef44d9..a7ddd62 100644 --- a/data-extraction/src/js/helpers/find.ts +++ b/data-extraction/src/js/helpers/find.ts @@ -42,3 +42,95 @@ export function find(predicate: (obj: unknown) => boolean): unknown { return undefined; } + +export function findVar( + options: { nameSimilarTo?: string; ctor?: string }, + predicate?: (value: any) => boolean +): unknown | undefined { + if (!DataExtractorApiImpl.lastContext) { + throw new Error("No data extractor context!"); + } + + let bestValue = undefined; + let bestValueScore = undefined; // minimized + + for (const [name, value] of Object.entries( + DataExtractorApiImpl.lastContext.variablesInScope + )) { + const v = value(); + if (options.ctor !== undefined) { + if ( + typeof v !== "object" || + !v || + v.constructor.name !== options.ctor + ) { + continue; + } + } + if (predicate) { + if (!predicate(v)) { + continue; + } + } + let score = 0; + if (options.nameSimilarTo !== undefined) { + score += similarityScore(name, options.nameSimilarTo); + } else { + return v; + } + if (bestValueScore === undefined || score < bestValueScore) { + bestValue = v; + bestValueScore = score; + } + } + + return bestValue; +} + +function similarityScore(a: string, b: string): number { + const distance = levenshteinDistance(a, b); + + const aSorted = a.split("").sort().join(""); + const bSorted = b.split("").sort().join(""); + const distance2 = levenshteinDistance(aSorted, bSorted); + + return distance * 10 + distance2; +} + +function levenshteinDistance(a: string, b: string): number { + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + + const matrix = []; + + // increment along the first column of each row + let i; + for (i = 0; i <= b.length; i++) { + matrix[i] = [i]; + } + + // increment each column in the first row + let j; + for (j = 0; j <= a.length; j++) { + matrix[0][j] = j; + } + + // Fill in the rest of the matrix + for (i = 1; i <= b.length; i++) { + for (j = 1; j <= a.length; j++) { + if (b.charAt(i - 1) == a.charAt(j - 1)) { + matrix[i][j] = matrix[i - 1][j - 1]; + } else { + matrix[i][j] = Math.min( + matrix[i - 1][j - 1] + 1, // substitution + Math.min( + matrix[i][j - 1] + 1, // insertion + matrix[i - 1][j] + 1 + ) + ); // deletion + } + } + } + + return matrix[b.length][a.length]; +} diff --git a/extension/src/VisualizationBackend/JsVisualizationSupport.ts b/extension/src/VisualizationBackend/JsVisualizationSupport.ts index 0abd123..0742393 100644 --- a/extension/src/VisualizationBackend/JsVisualizationSupport.ts +++ b/extension/src/VisualizationBackend/JsVisualizationSupport.ts @@ -146,7 +146,7 @@ class JsVisualizationBackend extends VisualizationBackendBase { : "undefined"; const body = `${getExpressionForDataExtractorApi()}.getData( - e => (${expression}), + () => (${expression}), expr => eval(expr), ${preferredExtractorExpr}, {${variableNames.map((n) => `${n}: () => ${n}`).join(",")}}, @@ -181,7 +181,7 @@ class JsVisualizationBackend extends VisualizationBackendBase { if (result.kind === "NoExtractors") { throw new Error("No extractors"); } else if (result.kind === "Error") { - throw new Error(result.message); + throw new Error(result.message + "\n" + (result as any).stack); } else if (result.kind === "Data") { return { kind: "data",