🎉 initiate project *astro_rewrite*

This commit is contained in:
sindrekjelsrud 2023-07-19 21:31:30 +02:00
parent ffd4d5e86c
commit 2ba37bfbe3
8658 changed files with 2268794 additions and 2538 deletions

View file

@ -0,0 +1,24 @@
import { OptionalStandardTokenType } from "../encodedTokenAttributes";
import { IEmbeddedLanguagesMap } from "../main";
import { ScopeName } from "../theme";
export declare class BasicScopeAttributes {
readonly languageId: number;
readonly tokenType: OptionalStandardTokenType;
constructor(languageId: number, tokenType: OptionalStandardTokenType);
}
export declare class BasicScopeAttributesProvider {
private readonly _defaultAttributes;
private readonly _embeddedLanguagesMatcher;
constructor(initialLanguageId: number, embeddedLanguages: IEmbeddedLanguagesMap | null);
getDefaultAttributes(): BasicScopeAttributes;
getBasicScopeAttributes(scopeName: ScopeName | null): BasicScopeAttributes;
private static readonly _NULL_SCOPE_METADATA;
private readonly _getBasicScopeAttributes;
/**
* Given a produced TM scope, return the language that token describes or null if unknown.
* e.g. source.html => html, source.css.embedded.html => css, punctuation.definition.tag.html => null
*/
private _scopeToLanguage;
private _toStandardTokenType;
private static STANDARD_TOKEN_TYPE_REGEXP;
}

View file

@ -0,0 +1,198 @@
import { EncodedTokenAttributes, StandardTokenType } from '../encodedTokenAttributes';
import { IEmbeddedLanguagesMap, IGrammar, IToken, ITokenizeLineResult, ITokenizeLineResult2, ITokenTypeMap, StateStack as StackElementDef } from '../main';
import { Matcher } from '../matcher';
import { IOnigLib, OnigScanner, OnigString } from '../onigLib';
import { IRawGrammar, IRawRepository } from '../rawGrammar';
import { IRuleFactoryHelper, IRuleRegistry, Rule, RuleId } from '../rule';
import { ScopeName, ScopePath, ScopeStack, StyleAttributes } from '../theme';
import { BasicScopeAttributes } from './basicScopesAttributeProvider';
export declare function createGrammar(scopeName: ScopeName, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, balancedBracketSelectors: BalancedBracketSelectors | null, grammarRepository: IGrammarRepository & IThemeProvider, onigLib: IOnigLib): Grammar;
export interface IThemeProvider {
themeMatch(scopePath: ScopeStack): StyleAttributes | null;
getDefaults(): StyleAttributes;
}
export interface IGrammarRepository {
lookup(scopeName: ScopeName): IRawGrammar | undefined;
injections(scopeName: ScopeName): ScopeName[];
}
export interface Injection {
readonly debugSelector: string;
readonly matcher: Matcher<string[]>;
readonly priority: -1 | 0 | 1;
readonly ruleId: RuleId;
readonly grammar: IRawGrammar;
}
export declare class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib {
private readonly _rootScopeName;
private readonly balancedBracketSelectors;
private readonly _onigLib;
private _rootId;
private _lastRuleId;
private readonly _ruleId2desc;
private readonly _includedGrammars;
private readonly _grammarRepository;
private readonly _grammar;
private _injections;
private readonly _basicScopeAttributesProvider;
private readonly _tokenTypeMatchers;
get themeProvider(): IThemeProvider;
constructor(_rootScopeName: ScopeName, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, balancedBracketSelectors: BalancedBracketSelectors | null, grammarRepository: IGrammarRepository & IThemeProvider, _onigLib: IOnigLib);
dispose(): void;
createOnigScanner(sources: string[]): OnigScanner;
createOnigString(sources: string): OnigString;
getMetadataForScope(scope: string): BasicScopeAttributes;
private _collectInjections;
getInjections(): Injection[];
registerRule<T extends Rule>(factory: (id: RuleId) => T): T;
getRule(ruleId: RuleId): Rule;
getExternalGrammar(scopeName: string, repository?: IRawRepository): IRawGrammar | undefined;
tokenizeLine(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult;
tokenizeLine2(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult2;
private _tokenize;
}
export declare class AttributedScopeStack {
readonly parent: AttributedScopeStack | null;
readonly scopePath: ScopeStack;
readonly tokenAttributes: EncodedTokenAttributes;
static createRoot(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes): AttributedScopeStack;
static createRootAndLookUpScopeName(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes, grammar: Grammar): AttributedScopeStack;
get scopeName(): ScopeName;
private constructor();
equals(other: AttributedScopeStack): boolean;
private static _equals;
private static mergeAttributes;
pushAttributed(scopePath: ScopePath | null, grammar: Grammar): AttributedScopeStack;
private static _pushAttributed;
getScopeNames(): string[];
}
/**
* Represents a "pushed" state on the stack (as a linked list element).
*/
export declare class StateStack implements StackElementDef {
/**
* The previous state on the stack (or null for the root state).
*/
readonly parent: StateStack | null;
/**
* The state (rule) that this element represents.
*/
private readonly ruleId;
/**
* The state has entered and captured \n. This means that the next line should have an anchorPosition of 0.
*/
readonly beginRuleCapturedEOL: boolean;
/**
* The "pop" (end) condition for this state in case that it was dynamically generated through captured text.
*/
readonly endRule: string | null;
/**
* The list of scopes containing the "name" for this state.
*/
readonly nameScopesList: AttributedScopeStack;
/**
* The list of scopes containing the "contentName" (besides "name") for this state.
* This list **must** contain as an element `scopeName`.
*/
readonly contentNameScopesList: AttributedScopeStack;
_stackElementBrand: void;
static NULL: StateStack;
/**
* The position on the current line where this state was pushed.
* This is relevant only while tokenizing a line, to detect endless loops.
* Its value is meaningless across lines.
*/
private _enterPos;
/**
* The captured anchor position when this stack element was pushed.
* This is relevant only while tokenizing a line, to restore the anchor position when popping.
* Its value is meaningless across lines.
*/
private _anchorPos;
/**
* The depth of the stack.
*/
readonly depth: number;
constructor(
/**
* The previous state on the stack (or null for the root state).
*/
parent: StateStack | null,
/**
* The state (rule) that this element represents.
*/
ruleId: RuleId, enterPos: number, anchorPos: number,
/**
* The state has entered and captured \n. This means that the next line should have an anchorPosition of 0.
*/
beginRuleCapturedEOL: boolean,
/**
* The "pop" (end) condition for this state in case that it was dynamically generated through captured text.
*/
endRule: string | null,
/**
* The list of scopes containing the "name" for this state.
*/
nameScopesList: AttributedScopeStack,
/**
* The list of scopes containing the "contentName" (besides "name") for this state.
* This list **must** contain as an element `scopeName`.
*/
contentNameScopesList: AttributedScopeStack);
equals(other: StateStack): boolean;
private static _equals;
/**
* A structural equals check. Does not take into account `scopes`.
*/
private static _structuralEquals;
clone(): StateStack;
private static _reset;
reset(): void;
pop(): StateStack | null;
safePop(): StateStack;
push(ruleId: RuleId, enterPos: number, anchorPos: number, beginRuleCapturedEOL: boolean, endRule: string | null, nameScopesList: AttributedScopeStack, contentNameScopesList: AttributedScopeStack): StateStack;
getEnterPos(): number;
getAnchorPos(): number;
getRule(grammar: IRuleRegistry): Rule;
toString(): string;
private _writeString;
withContentNameScopesList(contentNameScopeStack: AttributedScopeStack): StateStack;
withEndRule(endRule: string): StateStack;
hasSameRuleAs(other: StateStack): boolean;
}
interface TokenTypeMatcher {
readonly matcher: Matcher<string[]>;
readonly type: StandardTokenType;
}
export declare class BalancedBracketSelectors {
private readonly balancedBracketScopes;
private readonly unbalancedBracketScopes;
private allowAny;
constructor(balancedBracketScopes: string[], unbalancedBracketScopes: string[]);
get matchesAlways(): boolean;
get matchesNever(): boolean;
match(scopes: string[]): boolean;
}
export declare class LineTokens {
private readonly balancedBracketSelectors;
private readonly _emitBinaryTokens;
/**
* defined only if `DebugFlags.InDebugMode`.
*/
private readonly _lineText;
/**
* used only if `_emitBinaryTokens` is false.
*/
private readonly _tokens;
/**
* used only if `_emitBinaryTokens` is true.
*/
private readonly _binaryTokens;
private _lastTokenEndIndex;
private readonly _tokenTypeOverrides;
constructor(emitBinaryTokens: boolean, lineText: string, tokenTypeOverrides: TokenTypeMatcher[], balancedBracketSelectors: BalancedBracketSelectors | null);
produce(stack: StateStack, endIndex: number): void;
produceFromScopes(scopesList: AttributedScopeStack, endIndex: number): void;
getResult(stack: StateStack, lineLength: number): IToken[];
getBinaryResult(stack: StateStack, lineLength: number): Uint32Array;
}
export {};

View file

@ -0,0 +1,68 @@
import { IRawRule } from '../rawGrammar';
import { ScopeName } from '../theme';
import { IGrammarRepository } from './grammar';
export declare type AbsoluteRuleReference = TopLevelRuleReference | TopLevelRepositoryRuleReference;
/**
* References the top level rule of a grammar with the given scope name.
*/
export declare class TopLevelRuleReference {
readonly scopeName: ScopeName;
constructor(scopeName: ScopeName);
toKey(): string;
}
/**
* References a rule of a grammar in the top level repository section with the given name.
*/
export declare class TopLevelRepositoryRuleReference {
readonly scopeName: ScopeName;
readonly ruleName: string;
constructor(scopeName: ScopeName, ruleName: string);
toKey(): string;
}
export declare class ExternalReferenceCollector {
private readonly _references;
private readonly _seenReferenceKeys;
get references(): readonly AbsoluteRuleReference[];
readonly visitedRule: Set<IRawRule>;
add(reference: AbsoluteRuleReference): void;
}
export declare class ScopeDependencyProcessor {
readonly repo: IGrammarRepository;
readonly initialScopeName: ScopeName;
readonly seenFullScopeRequests: Set<string>;
readonly seenPartialScopeRequests: Set<string>;
Q: AbsoluteRuleReference[];
constructor(repo: IGrammarRepository, initialScopeName: ScopeName);
processQueue(): void;
}
export declare type IncludeReference = BaseReference | SelfReference | RelativeReference | TopLevelReference | TopLevelRepositoryReference;
export declare const enum IncludeReferenceKind {
Base = 0,
Self = 1,
RelativeReference = 2,
TopLevelReference = 3,
TopLevelRepositoryReference = 4
}
export declare class BaseReference {
readonly kind = IncludeReferenceKind.Base;
}
export declare class SelfReference {
readonly kind = IncludeReferenceKind.Self;
}
export declare class RelativeReference {
readonly ruleName: string;
readonly kind = IncludeReferenceKind.RelativeReference;
constructor(ruleName: string);
}
export declare class TopLevelReference {
readonly scopeName: ScopeName;
readonly kind = IncludeReferenceKind.TopLevelReference;
constructor(scopeName: ScopeName);
}
export declare class TopLevelRepositoryReference {
readonly scopeName: ScopeName;
readonly ruleName: string;
readonly kind = IncludeReferenceKind.TopLevelRepositoryReference;
constructor(scopeName: ScopeName, ruleName: string);
}
export declare function parseInclude(include: string): IncludeReference;

View file

@ -0,0 +1 @@
export * from './grammar';

View file

@ -0,0 +1,27 @@
import type { LineTokens, StateStack } from './grammar';
import { OnigString } from '../onigLib';
import type { AttributedScopeStack, Grammar } from './grammar';
declare class TokenizeStringResult {
readonly stack: StateStack;
readonly stoppedEarly: boolean;
constructor(stack: StateStack, stoppedEarly: boolean);
}
/**
* Tokenize a string
* @param grammar
* @param lineText
* @param isFirstLine
* @param linePos
* @param stack
* @param lineTokens
* @param checkWhileConditions
* @param timeLimit Use `0` to indicate no time limit
* @returns the StackElement or StackElement.TIME_LIMIT_REACHED if the time limit has been reached
*/
export declare function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StateStack, lineTokens: LineTokens, checkWhileConditions: boolean, timeLimit: number): TokenizeStringResult;
export declare class LocalStackElement {
readonly scopes: AttributedScopeStack;
readonly endPos: number;
constructor(scopes: AttributedScopeStack, endPos: number);
}
export {};