Data extraction API improvements.

data-extraction/src/js/api/DataExtractorApi.ts

@@ -53,6 +53,11 @@ export interface DataExtractor {
 	 * Must be unique among all data extractors.
 	 */
 	id: string;
+
+	/**
+	 * Filters the data to be extracted.
+	 */
+	dataCtor?: string;
 	getExtractions(
 		data: unknown,
 		extractionCollector: ExtractionCollector,
@@ -73,7 +78,11 @@ export interface DataExtractorContext {
 	 */
 	evalFn: <TEval>(expression: string) => TEval;
 
+	expression: string | undefined;
+
 	variablesInScope: Record<string, () => unknown>;
+
+	extract(value: unknown): VisualizationData | undefined;
 }
 
 export interface DataExtraction {
@@ -86,10 +95,12 @@ export interface DataExtraction {
 	 * A unique id identifying this extraction among all extractions.
 	 * Required to express extraction preferences.
 	 */
-	id: string;
+	id?: string;
+
 	/**
 	 * A user friendly name of this extraction.
 	 */
-	name: string;
+	name?: string;
+
 	extractData(): VisualizationData;
 }

data-extraction/src/js/api/DataExtractorApiImpl.ts

@@ -7,7 +7,10 @@ import {
 	ExtractionCollector,
 	DataExtractorContext,
 } from "./DataExtractorApi";
-import { DataExtractorInfo } from "../../DataExtractionResult";
+import {
+	DataExtractorInfo,
+	VisualizationData,
+} from "../../DataExtractionResult";
 import { registerDefaultExtractors } from "./default-extractors";
 import { LoadDataExtractorsFn } from "./LoadDataExtractorsFn";
 import * as helpers from "../helpers";
@@ -41,36 +44,56 @@ export class DataExtractorApiImpl implements DataExtractorApi {
 		preferredDataExtractorId: string | undefined,
 		variablesInScope: Record<string, () => unknown>
 	): JSONString<DataResult> {
-		const extractions = new Array<DataExtraction>();
-		const extractionCollector: ExtractionCollector = {
-			addExtraction(extraction) {
-				extractions.push(extraction);
-			},
-		};
+		class ContextImpl implements DataExtractorContext {
+			constructor(
+				public readonly variablesInScope: Record<string, () => unknown>,
+				public readonly expression: string | undefined,
+				public readonly evalFn: <T>(expression: string) => T,
+				private readonly _api: DataExtractorApiImpl,
+				private readonly _parent: ContextImpl | undefined
+			) {}
 
-		const context: DataExtractorContext = {
-			evalFn,
+			get _level(): number {
+				return this._parent ? this._parent._level + 1 : 0;
+			}
+
+			extract(value: any): VisualizationData | undefined {
+				if (this._level > 10) {
+					throw new Error(
+						"extract() called too many times recursively"
+					);
+				}
+
+				const extractions = this._api.getExtractions(
+					value,
+					new ContextImpl(
+						this.variablesInScope,
+						undefined,
+						this.evalFn,
+						this._api,
+						this
+					)
+				);
+				if (extractions.length === 0) {
+					return undefined;
+				}
+				return extractions[0].extractData();
+			}
+		}
+
+		const rootContext = new ContextImpl(
 			variablesInScope,
-		};
+			removeEnd(removeStart(valueFn.toString(), "() => ("), ")").trim(),
+			evalFn,
+			this,
+			undefined
+		);
 
-		DataExtractorApiImpl.lastContext = context;
+		DataExtractorApiImpl.lastContext = rootContext;
 		const value = valueFn();
-
-		const extractors = new Array<DataExtractor>();
-
-		for (const fn of this.extractorSources.values()) {
-			fn((extractor) => {
-				extractors.push(extractor);
-			}, helpers);
-		}
-
-		for (const e of [...this.extractors.values(), ...extractors]) {
-			e.getExtractions(value, extractionCollector, context);
-		}
-
+		const extractions = this.getExtractions(value, rootContext);
 		DataExtractorApiImpl.lastContext = undefined;
 
-		extractions.sort((a, b) => b.priority - a.priority);
 		let usedExtraction = extractions[0];
 		if (!usedExtraction) {
 			return this.toJson({ kind: "NoExtractors" } as DataResult);
@@ -87,8 +110,8 @@ export class DataExtractorApiImpl implements DataExtractorApi {
 
 		function mapExtractor(e: DataExtraction): DataExtractorInfo {
 			return {
-				id: e.id as any,
-				name: e.name,
+				id: e.id! as any,
+				name: e.name!,
 				priority: e.priority,
 			};
 		}
@@ -104,6 +127,50 @@ export class DataExtractorApiImpl implements DataExtractorApi {
 		} as DataResult);
 	}
 
+	public getExtractions(
+		value: unknown,
+		context: DataExtractorContext
+	): DataExtraction[] {
+		const extractions = new Array<DataExtraction>();
+		const extractors = new Array<DataExtractor>();
+
+		for (const fn of this.extractorSources.values()) {
+			fn((extractor) => {
+				extractors.push(extractor);
+			}, helpers);
+		}
+
+		for (const e of [...this.extractors.values(), ...extractors]) {
+			if (e.dataCtor !== undefined) {
+				if (
+					typeof value !== "object" ||
+					value === null ||
+					value.constructor.name !== e.dataCtor
+				) {
+					continue;
+				}
+			}
+			e.getExtractions(
+				value,
+				{
+					addExtraction(extraction) {
+						if (extraction.id === undefined) {
+							extraction.id = e.id;
+						}
+						if (extraction.name === undefined) {
+							extraction.name = e.id;
+						}
+						extractions.push(extraction);
+					},
+				},
+				context
+			);
+		}
+		extractions.sort((a, b) => b.priority - a.priority);
+
+		return extractions;
+	}
+
 	public registerDefaultExtractors(preferExisting: boolean = false): void {
 		// TODO consider preferExisting
 		registerDefaultExtractors(this);
@@ -120,3 +187,17 @@ export class DataExtractorApiImpl implements DataExtractorApi {
 		}
 	}
 }
+
+function removeStart(str: string, start: string): string {
+	if (str.startsWith(start)) {
+		return str.substr(start.length);
+	}
+	return str;
+}
+
+function removeEnd(str: string, end: string): string {
+	if (str.endsWith(end)) {
+		return str.substr(0, str.length - end.length);
+	}
+	return str;
+}

data-extraction/src/js/api/default-extractors/StringRangeExtractor.ts

@@ -50,7 +50,11 @@ export class StringRangeExtractor implements DataExtractor {
 
 		for (let item of (data as (number | [number, number])[]).slice(1)) {
 			if (typeof item === "string") {
-				item = context.evalFn(item);
+				try {
+					item = context.evalFn(item);
+				} catch (e) {
+					return;
+				}
 				if (item === undefined) {
 					continue;
 				}

data-extraction/src/js/helpers/find.ts

@@ -42,3 +42,95 @@ export function find(predicate: (obj: unknown) => boolean): unknown {
 
 	return undefined;
 }
+
+export function findVar(
+	options: { nameSimilarTo?: string; ctor?: string },
+	predicate?: (value: any) => boolean
+): unknown | undefined {
+	if (!DataExtractorApiImpl.lastContext) {
+		throw new Error("No data extractor context!");
+	}
+
+	let bestValue = undefined;
+	let bestValueScore = undefined; // minimized
+
+	for (const [name, value] of Object.entries(
+		DataExtractorApiImpl.lastContext.variablesInScope
+	)) {
+		const v = value();
+		if (options.ctor !== undefined) {
+			if (
+				typeof v !== "object" ||
+				!v ||
+				v.constructor.name !== options.ctor
+			) {
+				continue;
+			}
+		}
+		if (predicate) {
+			if (!predicate(v)) {
+				continue;
+			}
+		}
+		let score = 0;
+		if (options.nameSimilarTo !== undefined) {
+			score += similarityScore(name, options.nameSimilarTo);
+		} else {
+			return v;
+		}
+		if (bestValueScore === undefined || score < bestValueScore) {
+			bestValue = v;
+			bestValueScore = score;
+		}
+	}
+
+	return bestValue;
+}
+
+function similarityScore(a: string, b: string): number {
+	const distance = levenshteinDistance(a, b);
+
+	const aSorted = a.split("").sort().join("");
+	const bSorted = b.split("").sort().join("");
+	const distance2 = levenshteinDistance(aSorted, bSorted);
+
+	return distance * 10 + distance2;
+}
+
+function levenshteinDistance(a: string, b: string): number {
+	if (a.length === 0) return b.length;
+	if (b.length === 0) return a.length;
+
+	const matrix = [];
+
+	// increment along the first column of each row
+	let i;
+	for (i = 0; i <= b.length; i++) {
+		matrix[i] = [i];
+	}
+
+	// increment each column in the first row
+	let j;
+	for (j = 0; j <= a.length; j++) {
+		matrix[0][j] = j;
+	}
+
+	// Fill in the rest of the matrix
+	for (i = 1; i <= b.length; i++) {
+		for (j = 1; j <= a.length; j++) {
+			if (b.charAt(i - 1) == a.charAt(j - 1)) {
+				matrix[i][j] = matrix[i - 1][j - 1];
+			} else {
+				matrix[i][j] = Math.min(
+					matrix[i - 1][j - 1] + 1, // substitution
+					Math.min(
+						matrix[i][j - 1] + 1, // insertion
+						matrix[i - 1][j] + 1
+					)
+				); // deletion
+			}
+		}
+	}
+
+	return matrix[b.length][a.length];
+}

extension/src/VisualizationBackend/JsVisualizationSupport.ts

@@ -146,7 +146,7 @@ class JsVisualizationBackend extends VisualizationBackendBase {
 				: "undefined";
 
 			const body = `${getExpressionForDataExtractorApi()}.getData(
-                    e => (${expression}),
+                    () => (${expression}),
                     expr => eval(expr),
                     ${preferredExtractorExpr},
 					{${variableNames.map((n) => `${n}: () => ${n}`).join(",")}},
@@ -181,7 +181,7 @@ class JsVisualizationBackend extends VisualizationBackendBase {
 			if (result.kind === "NoExtractors") {
 				throw new Error("No extractors");
 			} else if (result.kind === "Error") {
-				throw new Error(result.message);
+				throw new Error(result.message + "\n" + (result as any).stack);
 			} else if (result.kind === "Data") {
 				return {
 					kind: "data",