vrr/ProxyAI
2
0
Fork 0
mirror of https://github.com/carlrobertoh/ProxyAI.git synced 2026-10-19 09:43:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: clean up old code

Browse source
This commit is contained in:
Carl-Robert Linnupuu 2024-03-14 14:31:16 +02:00
parent 33c597d8e5
commit 42105bf308
67 changed files with 75 additions and 8227 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/build.yml vendored
View file

@ -21,9 +21,6 @@ jobs:
java-version: '17'
distribution: 'temurin'
- name: Generate Grammar Source
run: ./gradlew generateGrammarSource
- name: Run Tests
run: ./gradlew check

1
build.gradle.kts
View file

@ -47,7 +47,6 @@ changelog {
}
dependencies {
implementation(project(":codegpt-core"))
implementation(project(":codegpt-telemetry"))
implementation(project(":codegpt-treesitter"))

19
codegpt-core/build.gradle.kts
View file

@ -1,19 +0,0 @@
plugins {
id("codegpt.java-conventions")
id("antlr")
}
dependencies {
antlr("org.antlr:antlr4:4.13.1")
api("com.github.jelmerk:hnswlib-core:1.1.2")
api("com.github.jelmerk:hnswlib-utils:1.1.2")
implementation("org.antlr:antlr4-runtime:4.13.1")
implementation("org.json:json:20240303")
}
tasks {
generateGrammarSource {
outputDirectory = file("src/main/java/grammar")
arguments = arguments + listOf("-package", "grammar")
}
}

82
codegpt-core/src/main/antlr/JSON.g4
View file

@ -1,82 +0,0 @@
/** Taken from "The Definitive ANTLR 4 Reference" by Terence Parr */
// Derived from https://json.org
grammar JSON;
json
: value EOF
;
obj
: '{' pair (',' pair)* '}'
| '{' '}'
;
pair
: STRING ':' value
;
arr
: '[' value (',' value)* ']'
| '[' ']'
;
value
: STRING
| NUMBER
| obj
| arr
| 'true'
| 'false'
| 'null'
;
STRING
: '"' (ESC | SAFECODEPOINT)* '"'
;
fragment ESC
: '\\' (["\\/bfnrt] | UNICODE)
;
fragment UNICODE
: 'u' HEX HEX HEX HEX
;
fragment HEX
: [0-9a-fA-F]
;
fragment SAFECODEPOINT
: ~ ["\\\u0000-\u001F]
;
NUMBER
: '-'? INT ('.' [0-9] +)? EXP?
;
fragment INT
// integer part forbis leading 0s (e.g. `01`)
: '0' | [1-9] [0-9]*
;
// no leading zeros
fragment EXP
// exponent number permits leading 0s (e.g. `1e01`)
: [Ee] [+\-]? [0-9]+
;
// \- since - means "range" inside [...]
WS
: [ \t\n\r] + -> skip
;

241
codegpt-core/src/main/antlr/JavaLexer.g4
View file

@ -1,241 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2013 Terence Parr, Sam Harwell
Copyright (c) 2017 Ivan Kochurkin (upgrade to Java 8)
Copyright (c) 2021 Michał Lorek (upgrade to Java 11)
Copyright (c) 2022 Michał Lorek (upgrade to Java 17)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
lexer grammar JavaLexer;
// Keywords
ABSTRACT: 'abstract';
ASSERT: 'assert';
BOOLEAN: 'boolean';
BREAK: 'break';
BYTE: 'byte';
CASE: 'case';
CATCH: 'catch';
CHAR: 'char';
CLASS: 'class';
CONST: 'const';
CONTINUE: 'continue';
DEFAULT: 'default';
DO: 'do';
DOUBLE: 'double';
ELSE: 'else';
ENUM: 'enum';
EXTENDS: 'extends';
FINAL: 'final';
FINALLY: 'finally';
FLOAT: 'float';
FOR: 'for';
IF: 'if';
GOTO: 'goto';
IMPLEMENTS: 'implements';
IMPORT: 'import';
INSTANCEOF: 'instanceof';
INT: 'int';
INTERFACE: 'interface';
LONG: 'long';
NATIVE: 'native';
NEW: 'new';
PACKAGE: 'package';
PRIVATE: 'private';
PROTECTED: 'protected';
PUBLIC: 'public';
RETURN: 'return';
SHORT: 'short';
STATIC: 'static';
STRICTFP: 'strictfp';
SUPER: 'super';
SWITCH: 'switch';
SYNCHRONIZED: 'synchronized';
THIS: 'this';
THROW: 'throw';
THROWS: 'throws';
TRANSIENT: 'transient';
TRY: 'try';
VOID: 'void';
VOLATILE: 'volatile';
WHILE: 'while';
// Module related keywords
MODULE: 'module';
OPEN: 'open';
REQUIRES: 'requires';
EXPORTS: 'exports';
OPENS: 'opens';
TO: 'to';
USES: 'uses';
PROVIDES: 'provides';
WITH: 'with';
TRANSITIVE: 'transitive';
// Local Variable Type Inference
VAR: 'var'; // reserved type name
// Switch Expressions
YIELD: 'yield'; // reserved type name from Java 14
// Records
RECORD: 'record';
// Sealed Classes
SEALED: 'sealed';
PERMITS: 'permits';
NON_SEALED: 'non-sealed';
// Literals
DECIMAL_LITERAL: ('0' | [1-9] (Digits? | '_'+ Digits)) [lL]?;
HEX_LITERAL: '0' [xX] [0-9a-fA-F] ([0-9a-fA-F_]* [0-9a-fA-F])? [lL]?;
OCT_LITERAL: '0' '_'* [0-7] ([0-7_]* [0-7])? [lL]?;
BINARY_LITERAL: '0' [bB] [01] ([01_]* [01])? [lL]?;
FLOAT_LITERAL: (Digits '.' Digits? | '.' Digits) ExponentPart? [fFdD]?
| Digits (ExponentPart [fFdD]? | [fFdD])
;
HEX_FLOAT_LITERAL: '0' [xX] (HexDigits '.'? | HexDigits? '.' HexDigits) [pP] [+-]? Digits [fFdD]?;
BOOL_LITERAL: 'true'
| 'false'
;
CHAR_LITERAL: '\'' (~['\\\r\n] | EscapeSequence) '\'';
STRING_LITERAL: '"' (~["\\\r\n] | EscapeSequence)* '"';
TEXT_BLOCK: '"""' [ \t]* [\r\n] (. | EscapeSequence)*? '"""';
NULL_LITERAL: 'null';
// Separators
LPAREN: '(';
RPAREN: ')';
LBRACE: '{';
RBRACE: '}';
LBRACK: '[';
RBRACK: ']';
SEMI: ';';
COMMA: ',';
DOT: '.';
// Operators
ASSIGN: '=';
GT: '>';
LT: '<';
BANG: '!';
TILDE: '~';
QUESTION: '?';
COLON: ':';
EQUAL: '==';
LE: '<=';
GE: '>=';
NOTEQUAL: '!=';
AND: '&&';
OR: '||';
INC: '++';
DEC: '--';
ADD: '+';
SUB: '-';
MUL: '*';
DIV: '/';
BITAND: '&';
BITOR: '|';
CARET: '^';
MOD: '%';
ADD_ASSIGN: '+=';
SUB_ASSIGN: '-=';
MUL_ASSIGN: '*=';
DIV_ASSIGN: '/=';
AND_ASSIGN: '&=';
OR_ASSIGN: '|=';
XOR_ASSIGN: '^=';
MOD_ASSIGN: '%=';
LSHIFT_ASSIGN: '<<=';
RSHIFT_ASSIGN: '>>=';
URSHIFT_ASSIGN: '>>>=';
// Java 8 tokens
ARROW: '->';
COLONCOLON: '::';
// Additional symbols not defined in the lexical specification
AT: '@';
ELLIPSIS: '...';
// Whitespace and comments
WS: [ \t\r\n\u000C]+ -> channel(HIDDEN);
COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
LINE_COMMENT: '//' ~[\r\n]* -> channel(HIDDEN);
// Identifiers
IDENTIFIER: Letter LetterOrDigit*;
// Fragment rules
fragment ExponentPart
: [eE] [+-]? Digits
;
fragment EscapeSequence
: '\\' 'u005c'? [btnfr"'\\]
| '\\' 'u005c'? ([0-3]? [0-7])? [0-7]
| '\\' 'u'+ HexDigit HexDigit HexDigit HexDigit
;
fragment HexDigits
: HexDigit ((HexDigit | '_')* HexDigit)?
;
fragment HexDigit
: [0-9a-fA-F]
;
fragment Digits
: [0-9] ([0-9_]* [0-9])?
;
fragment LetterOrDigit
: Letter
| [0-9]
;
fragment Letter
: [a-zA-Z$_] // these are the "java letters" below 0x7F
| ~[\u0000-\u007F\uD800-\uDBFF] // covers all characters above 0x7F which are not a surrogate
| [\uD800-\uDBFF] [\uDC00-\uDFFF] // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
;

780
codegpt-core/src/main/antlr/JavaParser.g4
View file

@ -1,780 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2013 Terence Parr, Sam Harwell
Copyright (c) 2017 Ivan Kochurkin (upgrade to Java 8)
Copyright (c) 2021 Michał Lorek (upgrade to Java 11)
Copyright (c) 2022 Michał Lorek (upgrade to Java 17)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
parser grammar JavaParser;
options { tokenVocab=JavaLexer; }
compilationUnit
: packageDeclaration? (importDeclaration | ';')* (typeDeclaration | ';')*
| moduleDeclaration EOF
;
packageDeclaration
: annotation* PACKAGE qualifiedName ';'
;
importDeclaration
: IMPORT STATIC? qualifiedName ('.' '*')? ';'
;
typeDeclaration
: classOrInterfaceModifier*
(classDeclaration | enumDeclaration | interfaceDeclaration | annotationTypeDeclaration | recordDeclaration)
;
modifier
: classOrInterfaceModifier
| NATIVE
| SYNCHRONIZED
| TRANSIENT
| VOLATILE
;
classOrInterfaceModifier
: annotation
| PUBLIC
| PROTECTED
| PRIVATE
| STATIC
| ABSTRACT
| FINAL // FINAL for class only -- does not apply to interfaces
| STRICTFP
| SEALED // Java17
| NON_SEALED // Java17
;
variableModifier
: FINAL
| annotation
;
classDeclaration
: CLASS identifier typeParameters?
(EXTENDS typeType)?
(IMPLEMENTS typeList)?
(PERMITS typeList)? // Java17
classBody
;
typeParameters
: '<' typeParameter (',' typeParameter)* '>'
;
typeParameter
: annotation* identifier (EXTENDS annotation* typeBound)?
;
typeBound
: typeType ('&' typeType)*
;
enumDeclaration
: ENUM identifier (IMPLEMENTS typeList)? '{' enumConstants? ','? enumBodyDeclarations? '}'
;
enumConstants
: enumConstant (',' enumConstant)*
;
enumConstant
: annotation* identifier arguments? classBody?
;
enumBodyDeclarations
: ';' classBodyDeclaration*
;
interfaceDeclaration
: INTERFACE identifier typeParameters? (EXTENDS typeList)? (PERMITS typeList)? interfaceBody
;
classBody
: '{' classBodyDeclaration* '}'
;
interfaceBody
: '{' interfaceBodyDeclaration* '}'
;
classBodyDeclaration
: ';'
| STATIC? block
| modifier* memberDeclaration
;
memberDeclaration
: recordDeclaration //Java17
| methodDeclaration
| genericMethodDeclaration
| fieldDeclaration
| constructorDeclaration
| genericConstructorDeclaration
| interfaceDeclaration
| annotationTypeDeclaration
| classDeclaration
| enumDeclaration
;
/* We use rule this even for void methods which cannot have [] after parameters.
This simplifies grammar and we can consider void to be a type, which
renders the [] matching as a context-sensitive issue or a semantic check
for invalid return type after parsing.
*/
methodDeclaration
: typeTypeOrVoid identifier formalParameters ('[' ']')*
(THROWS qualifiedNameList)?
methodBody
;
methodBody
: block
| ';'
;
typeTypeOrVoid
: typeType
| VOID
;
genericMethodDeclaration
: typeParameters methodDeclaration
;
genericConstructorDeclaration
: typeParameters constructorDeclaration
;
constructorDeclaration
: identifier formalParameters (THROWS qualifiedNameList)? constructorBody=block
;
compactConstructorDeclaration
: modifier* identifier constructorBody=block
;
fieldDeclaration
: typeType variableDeclarators ';'
;
interfaceBodyDeclaration
: modifier* interfaceMemberDeclaration
| ';'
;
interfaceMemberDeclaration
: recordDeclaration // Java17
| constDeclaration
| interfaceMethodDeclaration
| genericInterfaceMethodDeclaration
| interfaceDeclaration
| annotationTypeDeclaration
| classDeclaration
| enumDeclaration
;
constDeclaration
: typeType constantDeclarator (',' constantDeclarator)* ';'
;
constantDeclarator
: identifier ('[' ']')* '=' variableInitializer
;
// Early versions of Java allows brackets after the method name, eg.
// public int[] return2DArray() [] { ... }
// is the same as
// public int[][] return2DArray() { ... }
interfaceMethodDeclaration
: interfaceMethodModifier* interfaceCommonBodyDeclaration
;
// Java8
interfaceMethodModifier
: annotation
| PUBLIC
| ABSTRACT
| DEFAULT
| STATIC
| STRICTFP
;
genericInterfaceMethodDeclaration
: interfaceMethodModifier* typeParameters interfaceCommonBodyDeclaration
;
interfaceCommonBodyDeclaration
: annotation* typeTypeOrVoid identifier formalParameters ('[' ']')* (THROWS qualifiedNameList)? methodBody
;
variableDeclarators
: variableDeclarator (',' variableDeclarator)*
;
variableDeclarator
: variableDeclaratorId ('=' variableInitializer)?
;
variableDeclaratorId
: identifier ('[' ']')*
;
variableInitializer
: arrayInitializer
| expression
;
arrayInitializer
: '{' (variableInitializer (',' variableInitializer)* ','? )? '}'
;
classOrInterfaceType
: (identifier typeArguments? '.')* typeIdentifier typeArguments?
;
typeArgument
: typeType
| annotation* '?' ((EXTENDS | SUPER) typeType)?
;
qualifiedNameList
: qualifiedName (',' qualifiedName)*
;
formalParameters
: '(' ( receiverParameter?
| receiverParameter (',' formalParameterList)?
| formalParameterList?
) ')'
;
receiverParameter
: typeType (identifier '.')* THIS
;
formalParameterList
: formalParameter (',' formalParameter)* (',' lastFormalParameter)?
| lastFormalParameter
;
formalParameter
: variableModifier* typeType variableDeclaratorId
;
lastFormalParameter
: variableModifier* typeType annotation* '...' variableDeclaratorId
;
// local variable type inference
lambdaLVTIList
: lambdaLVTIParameter (',' lambdaLVTIParameter)*
;
lambdaLVTIParameter
: variableModifier* VAR identifier
;
qualifiedName
: identifier ('.' identifier)*
;
literal
: integerLiteral
| floatLiteral
| CHAR_LITERAL
| STRING_LITERAL
| BOOL_LITERAL
| NULL_LITERAL
| TEXT_BLOCK // Java17
;
integerLiteral
: DECIMAL_LITERAL
| HEX_LITERAL
| OCT_LITERAL
| BINARY_LITERAL
;
floatLiteral
: FLOAT_LITERAL
| HEX_FLOAT_LITERAL
;
// ANNOTATIONS
altAnnotationQualifiedName
: (identifier DOT)* '@' identifier
;
annotation
: ('@' qualifiedName | altAnnotationQualifiedName) ('(' ( elementValuePairs | elementValue )? ')')?
;
elementValuePairs
: elementValuePair (',' elementValuePair)*
;
elementValuePair
: identifier '=' elementValue
;
elementValue
: expression
| annotation
| elementValueArrayInitializer
;
elementValueArrayInitializer
: '{' (elementValue (',' elementValue)*)? ','? '}'
;
annotationTypeDeclaration
: '@' INTERFACE identifier annotationTypeBody
;
annotationTypeBody
: '{' annotationTypeElementDeclaration* '}'
;
annotationTypeElementDeclaration
: modifier* annotationTypeElementRest
| ';' // this is not allowed by the grammar, but apparently allowed by the actual compiler
;
annotationTypeElementRest
: typeType annotationMethodOrConstantRest ';'
| classDeclaration ';'?
| interfaceDeclaration ';'?
| enumDeclaration ';'?
| annotationTypeDeclaration ';'?
| recordDeclaration ';'? // Java17
;
annotationMethodOrConstantRest
: annotationMethodRest
| annotationConstantRest
;
annotationMethodRest
: identifier '(' ')' defaultValue?
;
annotationConstantRest
: variableDeclarators
;
defaultValue
: DEFAULT elementValue
;
// MODULES - Java9
moduleDeclaration
: OPEN? MODULE qualifiedName moduleBody
;
moduleBody
: '{' moduleDirective* '}'
;
moduleDirective
: REQUIRES requiresModifier* qualifiedName ';'
| EXPORTS qualifiedName (TO qualifiedName)? ';'
| OPENS qualifiedName (TO qualifiedName)? ';'
| USES qualifiedName ';'
| PROVIDES qualifiedName WITH qualifiedName ';'
;
requiresModifier
: TRANSITIVE
| STATIC
;
// RECORDS - Java 17
recordDeclaration
: RECORD identifier typeParameters? recordHeader
(IMPLEMENTS typeList)?
recordBody
;
recordHeader
: '(' recordComponentList? ')'
;
recordComponentList
: recordComponent (',' recordComponent)*
;
recordComponent
: typeType identifier
;
recordBody
: '{' (classBodyDeclaration | compactConstructorDeclaration)* '}'
;
// STATEMENTS / BLOCKS
block
: '{' blockStatement* '}'
;
blockStatement
: localVariableDeclaration ';'
| localTypeDeclaration
| statement
;
localVariableDeclaration
: variableModifier* (VAR identifier '=' expression | typeType variableDeclarators)
;
identifier
: IDENTIFIER
| MODULE
| OPEN
| REQUIRES
| EXPORTS
| OPENS
| TO
| USES
| PROVIDES
| WITH
| TRANSITIVE
| YIELD
| SEALED
| PERMITS
| RECORD
| VAR
;
typeIdentifier // Identifiers that are not restricted for type declarations
: IDENTIFIER
| MODULE
| OPEN
| REQUIRES
| EXPORTS
| OPENS
| TO
| USES
| PROVIDES
| WITH
| TRANSITIVE
| SEALED
| PERMITS
| RECORD
;
localTypeDeclaration
: classOrInterfaceModifier*
(classDeclaration | interfaceDeclaration | recordDeclaration)
;
statement
: blockLabel=block
| ASSERT expression (':' expression)? ';'
| IF parExpression statement (ELSE statement)?
| FOR '(' forControl ')' statement
| WHILE parExpression statement
| DO statement WHILE parExpression ';'
| TRY block (catchClause+ finallyBlock? | finallyBlock)
| TRY resourceSpecification block catchClause* finallyBlock?
| SWITCH parExpression '{' switchBlockStatementGroup* switchLabel* '}'
| SYNCHRONIZED parExpression block
| RETURN expression? ';'
| THROW expression ';'
| BREAK identifier? ';'
| CONTINUE identifier? ';'
| YIELD expression ';' // Java17
| SEMI
| statementExpression=expression ';'
| switchExpression ';'? // Java17
| identifierLabel=identifier ':' statement
;
catchClause
: CATCH '(' variableModifier* catchType identifier ')' block
;
catchType
: qualifiedName ('|' qualifiedName)*
;
finallyBlock
: FINALLY block
;
resourceSpecification
: '(' resources ';'? ')'
;
resources
: resource (';' resource)*
;
resource
: variableModifier* ( classOrInterfaceType variableDeclaratorId | VAR identifier ) '=' expression
| qualifiedName
;
/** Matches cases then statements, both of which are mandatory.
* To handle empty cases at the end, we add switchLabel* to statement.
*/
switchBlockStatementGroup
: switchLabel+ blockStatement+
;
switchLabel
: CASE (constantExpression=expression | enumConstantName=IDENTIFIER | typeType varName=identifier) ':'
| DEFAULT ':'
;
forControl
: enhancedForControl
| forInit? ';' expression? ';' forUpdate=expressionList?
;
forInit
: localVariableDeclaration
| expressionList
;
enhancedForControl
: variableModifier* (typeType | VAR) variableDeclaratorId ':' expression
;
// EXPRESSIONS
parExpression
: '(' expression ')'
;
expressionList
: expression (',' expression)*
;
methodCall
: (identifier | THIS | SUPER) arguments
;
expression
// Expression order in accordance with https://introcs.cs.princeton.edu/java/11precedence/
// Level 16, Primary, array and member access
: primary
| expression '[' expression ']'
| expression bop='.'
(
identifier
| methodCall
| THIS
| NEW nonWildcardTypeArguments? innerCreator
| SUPER superSuffix
| explicitGenericInvocation
)
// Method calls and method references are part of primary, and hence level 16 precedence
| methodCall
| expression '::' typeArguments? identifier
| typeType '::' (typeArguments? identifier | NEW)
| classType '::' typeArguments? NEW
| switchExpression // Java17
// Level 15 Post-increment/decrement operators
| expression postfix=('++' | '--')
// Level 14, Unary operators
| prefix=('+'|'-'|'++'|'--'|'~'|'!') expression
// Level 13 Cast and object creation
| '(' annotation* typeType ('&' typeType)* ')' expression
| NEW creator
// Level 12 to 1, Remaining operators
| expression bop=('*'|'/'|'%') expression // Level 12, Multiplicative operators
| expression bop=('+'|'-') expression // Level 11, Additive operators
| expression ('<' '<' | '>' '>' '>' | '>' '>') expression // Level 10, Shift operators
| expression bop=('<=' | '>=' | '>' | '<') expression // Level 9, Relational operators
| expression bop=INSTANCEOF (typeType | pattern)
| expression bop=('==' | '!=') expression // Level 8, Equality Operators
| expression bop='&' expression // Level 7, Bitwise AND
| expression bop='^' expression // Level 6, Bitwise XOR
| expression bop='|' expression // Level 5, Bitwise OR
| expression bop='&&' expression // Level 4, Logic AND
| expression bop='||' expression // Level 3, Logic OR
| <assoc=right> expression bop='?' expression ':' expression // Level 2, Ternary
// Level 1, Assignment
| <assoc=right> expression
bop=('=' | '+=' | '-=' | '*=' | '/=' | '&=' | '|=' | '^=' | '>>=' | '>>>=' | '<<=' | '%=')
expression
// Level 0, Lambda Expression
| lambdaExpression // Java8
;
// Java17
pattern
: variableModifier* typeType annotation* identifier
;
// Java8
lambdaExpression
: lambdaParameters '->' lambdaBody
;
// Java8
lambdaParameters
: identifier
| '(' formalParameterList? ')'
| '(' identifier (',' identifier)* ')'
| '(' lambdaLVTIList? ')'
;
// Java8
lambdaBody
: expression
| block
;
primary
: '(' expression ')'
| THIS
| SUPER
| literal
| identifier
| typeTypeOrVoid '.' CLASS
| nonWildcardTypeArguments (explicitGenericInvocationSuffix | THIS arguments)
;
// Java17
switchExpression
: SWITCH parExpression '{' switchLabeledRule* '}'
;
// Java17
switchLabeledRule
: CASE (expressionList | NULL_LITERAL | guardedPattern) (ARROW | COLON) switchRuleOutcome
| DEFAULT (ARROW | COLON) switchRuleOutcome
;
// Java17
guardedPattern
: '(' guardedPattern ')'
| variableModifier* typeType annotation* identifier ('&&' expression)*
| guardedPattern '&&' expression
;
// Java17
switchRuleOutcome
: block
| blockStatement*
;
classType
: (classOrInterfaceType '.')? annotation* identifier typeArguments?
;
creator
: nonWildcardTypeArguments? createdName classCreatorRest
| createdName arrayCreatorRest
;
createdName
: identifier typeArgumentsOrDiamond? ('.' identifier typeArgumentsOrDiamond?)*
| primitiveType
;
innerCreator
: identifier nonWildcardTypeArgumentsOrDiamond? classCreatorRest
;
arrayCreatorRest
: ('[' ']')+ arrayInitializer
| ('[' expression ']')+ ('[' ']')*
;
classCreatorRest
: arguments classBody?
;
explicitGenericInvocation
: nonWildcardTypeArguments explicitGenericInvocationSuffix
;
typeArgumentsOrDiamond
: '<' '>'
| typeArguments
;
nonWildcardTypeArgumentsOrDiamond
: '<' '>'
| nonWildcardTypeArguments
;
nonWildcardTypeArguments
: '<' typeList '>'
;
typeList
: typeType (',' typeType)*
;
typeType
: annotation* (classOrInterfaceType | primitiveType) (annotation* '[' ']')*
;
primitiveType
: BOOLEAN
| CHAR
| BYTE
| SHORT
| INT
| LONG
| FLOAT
| DOUBLE
;
typeArguments
: '<' typeArgument (',' typeArgument)* '>'
;
superSuffix
: arguments
| '.' typeArguments? identifier arguments?
;
explicitGenericInvocationSuffix
: SUPER superSuffix
| identifier arguments
;
arguments
: '(' expressionList? ')'
;

587
codegpt-core/src/main/antlr/KotlinLexer.g4
View file

@ -1,587 +0,0 @@
/**
* Kotlin Grammar for ANTLR v4
*
* Based on:
* http://jetbrains.github.io/kotlin-spec/#_grammars_and_parsing
* and
* http://kotlinlang.org/docs/reference/grammar.html
*
* Tested on
* https://github.com/JetBrains/kotlin/tree/master/compiler/testData/psi
*/
lexer grammar KotlinLexer;
import UnicodeClasses;
ShebangLine
: '#!' ~[\u000A\u000D]*
-> channel(HIDDEN)
;
DelimitedComment
: '/*' ( DelimitedComment | . )*? '*/'
-> channel(HIDDEN)
;
LineComment
: '//' ~[\u000A\u000D]*
-> channel(HIDDEN)
;
WS
: [\u0020\u0009\u000C]
-> skip
;
NL: '\u000A' | '\u000D' '\u000A' ;
//SEPARATORS & OPERATIONS
RESERVED: '...' ;
DOT: '.' ;
COMMA: ',' ;
LPAREN: '(' -> pushMode(Inside) ;
RPAREN: ')' ;
LSQUARE: '[' -> pushMode(Inside) ;
RSQUARE: ']' ;
LCURL: '{' ;
RCURL: '}' ;
MULT: '*' ;
MOD: '%' ;
DIV: '/' ;
ADD: '+' ;
SUB: '-' ;
INCR: '++' ;
DECR: '--' ;
CONJ: '&&' ;
DISJ: '||' ;
EXCL: '!' ;
COLON: ':' ;
SEMICOLON: ';' ;
ASSIGNMENT: '=' ;
ADD_ASSIGNMENT: '+=' ;
SUB_ASSIGNMENT: '-=' ;
MULT_ASSIGNMENT: '*=' ;
DIV_ASSIGNMENT: '/=' ;
MOD_ASSIGNMENT: '%=' ;
ARROW: '->' ;
DOUBLE_ARROW: '=>' ;
RANGE: '..' ;
COLONCOLON: '::' ;
Q_COLONCOLON: '?::' ;
DOUBLE_SEMICOLON: ';;' ;
HASH: '#' ;
AT: '@' ;
QUEST: '?' ;
ELVIS: '?:' ;
LANGLE: '<' ;
RANGLE: '>' ;
LE: '<=' ;
GE: '>=' ;
EXCL_EQ: '!=' ;
EXCL_EQEQ: '!==' ;
AS_SAFE: 'as?' ;
EQEQ: '==' ;
EQEQEQ: '===' ;
SINGLE_QUOTE: '\'' ;
//KEYWORDS
RETURN_AT: 'return@' Identifier ;
CONTINUE_AT: 'continue@' Identifier ;
BREAK_AT: 'break@' Identifier ;
FILE: '@file' ;
PACKAGE: 'package' ;
IMPORT: 'import' ;
CLASS: 'class' ;
INTERFACE: 'interface' ;
FUN: 'fun' ;
OBJECT: 'object' ;
VAL: 'val' ;
VAR: 'var' ;
TYPE_ALIAS: 'typealias' ;
CONSTRUCTOR: 'constructor' ;
BY: 'by' ;
COMPANION: 'companion' ;
INIT: 'init' ;
THIS: 'this' ;
SUPER: 'super' ;
TYPEOF: 'typeof' ;
WHERE: 'where' ;
IF: 'if' ;
ELSE: 'else' ;
WHEN: 'when' ;
TRY: 'try' ;
CATCH: 'catch' ;
FINALLY: 'finally' ;
FOR: 'for' ;
DO: 'do' ;
WHILE: 'while' ;
THROW: 'throw' ;
RETURN: 'return' ;
CONTINUE: 'continue' ;
BREAK: 'break' ;
AS: 'as' ;
IS: 'is' ;
IN: 'in' ;
NOT_IS: '!is' (WS | NL)+ ;
NOT_IN: '!in' (WS | NL)+ ;
OUT: 'out' ;
FIELD: '@field' ;
PROPERTY: '@property' ;
GET: '@get' ;
SET: '@set' ;
GETTER: 'get' ;
SETTER: 'set' ;
RECEIVER: '@receiver' ;
PARAM: '@param' ;
SETPARAM: '@setparam' ;
DELEGATE: '@delegate' ;
DYNAMIC: 'dynamic' ;
//MODIFIERS
PUBLIC: 'public' ;
PRIVATE: 'private' ;
PROTECTED: 'protected' ;
INTERNAL: 'internal' ;
ENUM: 'enum' ;
SEALED: 'sealed' ;
ANNOTATION: 'annotation' ;
DATA: 'data' ;
INNER: 'inner' ;
TAILREC: 'tailrec' ;
OPERATOR: 'operator' ;
INLINE: 'inline' ;
INFIX: 'infix' ;
EXTERNAL: 'external' ;
SUSPEND: 'suspend' ;
OVERRIDE: 'override' ;
ABSTRACT: 'abstract' ;
FINAL: 'final' ;
OPEN: 'open' ;
CONST: 'const' ;
LATEINIT: 'lateinit' ;
VARARG: 'vararg' ;
NOINLINE: 'noinline' ;
CROSSINLINE: 'crossinline' ;
REIFIED: 'reified' ;
//
QUOTE_OPEN: '"' -> pushMode(LineString) ;
TRIPLE_QUOTE_OPEN: '"""' -> pushMode(MultiLineString) ;
RealLiteral
: FloatLiteral
| DoubleLiteral
;
FloatLiteral
: (DoubleLiteral | IntegerLiteral) [fF]
;
DoubleLiteral
: ( (DecDigitNoZero DecDigit* | '0')? '.'
| (DecDigitNoZero (DecDigit | '_')* DecDigit)? '.')
( DecDigit+
| DecDigit (DecDigit | '_')+ DecDigit
| DecDigit+ [eE] ('+' | '-')? DecDigit+
| DecDigit+ [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit
| DecDigit (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit+
| DecDigit (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit
)
;
LongLiteral
: (IntegerLiteral | HexLiteral | BinLiteral) 'L'
;
IntegerLiteral
: ('0'
| DecDigitNoZero DecDigit*
| DecDigitNoZero (DecDigit | '_')+ DecDigit
| DecDigitNoZero DecDigit* [eE] ('+' | '-')? DecDigit+
| DecDigitNoZero DecDigit* [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit
| DecDigitNoZero (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit+
| DecDigitNoZero (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit
)
;
fragment DecDigit
: UNICODE_CLASS_ND
;
fragment DecDigitNoZero
: UNICODE_CLASS_ND_NoZeros
;
fragment UNICODE_CLASS_ND_NoZeros
: '\u0031'..'\u0039'
| '\u0661'..'\u0669'
| '\u06f1'..'\u06f9'
| '\u07c1'..'\u07c9'
| '\u0967'..'\u096f'
| '\u09e7'..'\u09ef'
| '\u0a67'..'\u0a6f'
| '\u0ae7'..'\u0aef'
| '\u0b67'..'\u0b6f'
| '\u0be7'..'\u0bef'
| '\u0c67'..'\u0c6f'
| '\u0ce7'..'\u0cef'
| '\u0d67'..'\u0d6f'
| '\u0de7'..'\u0def'
| '\u0e51'..'\u0e59'
| '\u0ed1'..'\u0ed9'
| '\u0f21'..'\u0f29'
| '\u1041'..'\u1049'
| '\u1091'..'\u1099'
| '\u17e1'..'\u17e9'
| '\u1811'..'\u1819'
| '\u1947'..'\u194f'
| '\u19d1'..'\u19d9'
| '\u1a81'..'\u1a89'
| '\u1a91'..'\u1a99'
| '\u1b51'..'\u1b59'
| '\u1bb1'..'\u1bb9'
| '\u1c41'..'\u1c49'
| '\u1c51'..'\u1c59'
| '\ua621'..'\ua629'
| '\ua8d1'..'\ua8d9'
| '\ua901'..'\ua909'
| '\ua9d1'..'\ua9d9'
| '\ua9f1'..'\ua9f9'
| '\uaa51'..'\uaa59'
| '\uabf1'..'\uabf9'
| '\uff11'..'\uff19'
;
HexLiteral
: '0' [xX] HexDigit (HexDigit | '_')*
;
fragment HexDigit
: [0-9a-fA-F]
;
BinLiteral
: '0' [bB] BinDigit (BinDigit | '_')*
;
fragment BinDigit
: [01]
;
BooleanLiteral
: 'true'
| 'false'
;
NullLiteral
: 'null'
;
Identifier
: (Letter | '_') (Letter | '_' | DecDigit)*
| '`' ~('`')+ '`'
;
LabelReference
: '@' Identifier
;
LabelDefinition
: Identifier '@'
;
FieldIdentifier
: '$' Identifier
;
CharacterLiteral
: '\'' (EscapeSeq | .) '\''
;
fragment EscapeSeq
: UniCharacterLiteral
| EscapedIdentifier
;
fragment UniCharacterLiteral
: '\\' 'u' HexDigit HexDigit HexDigit HexDigit
;
fragment EscapedIdentifier
: '\\' ('t' | 'b' | 'r' | 'n' | '\'' | '"' | '\\' | '$')
;
fragment Letter
: UNICODE_CLASS_LL
| UNICODE_CLASS_LM
| UNICODE_CLASS_LO
| UNICODE_CLASS_LT
| UNICODE_CLASS_LU
| UNICODE_CLASS_NL
;
mode Inside ;
Inside_RPAREN: ')' -> popMode, type(RPAREN) ;
Inside_RSQUARE: ']' -> popMode, type(RSQUARE);
Inside_LPAREN: LPAREN -> pushMode(Inside), type(LPAREN) ;
Inside_LSQUARE: LSQUARE -> pushMode(Inside), type(LSQUARE) ;
Inside_LCURL: LCURL -> type(LCURL) ;
Inside_RCURL: RCURL -> type(RCURL) ;
Inside_DOT: DOT -> type(DOT) ;
Inside_COMMA: COMMA -> type(COMMA) ;
Inside_MULT: MULT -> type(MULT) ;
Inside_MOD: MOD -> type(MOD) ;
Inside_DIV: DIV -> type(DIV) ;
Inside_ADD: ADD -> type(ADD) ;
Inside_SUB: SUB -> type(SUB) ;
Inside_INCR: INCR -> type(INCR) ;
Inside_DECR: DECR -> type(DECR) ;
Inside_CONJ: CONJ -> type(CONJ) ;
Inside_DISJ: DISJ -> type(DISJ) ;
Inside_EXCL: EXCL -> type(EXCL) ;
Inside_COLON: COLON -> type(COLON) ;
Inside_SEMICOLON: SEMICOLON -> type(SEMICOLON) ;
Inside_ASSIGNMENT: ASSIGNMENT -> type(ASSIGNMENT) ;
Inside_ADD_ASSIGNMENT: ADD_ASSIGNMENT -> type(ADD_ASSIGNMENT) ;
Inside_SUB_ASSIGNMENT: SUB_ASSIGNMENT -> type(SUB_ASSIGNMENT) ;
Inside_MULT_ASSIGNMENT: MULT_ASSIGNMENT -> type(MULT_ASSIGNMENT) ;
Inside_DIV_ASSIGNMENT: DIV_ASSIGNMENT -> type(DIV_ASSIGNMENT) ;
Inside_MOD_ASSIGNMENT: MOD_ASSIGNMENT -> type(MOD_ASSIGNMENT) ;
Inside_ARROW: ARROW -> type(ARROW) ;
Inside_DOUBLE_ARROW: DOUBLE_ARROW -> type(DOUBLE_ARROW) ;
Inside_RANGE: RANGE -> type(RANGE) ;
Inside_RESERVED: RESERVED -> type(RESERVED) ;
Inside_COLONCOLON: COLONCOLON -> type(COLONCOLON) ;
Inside_Q_COLONCOLON: Q_COLONCOLON -> type(Q_COLONCOLON) ;
Inside_DOUBLE_SEMICOLON: DOUBLE_SEMICOLON -> type(DOUBLE_SEMICOLON) ;
Inside_HASH: HASH -> type(HASH) ;
Inside_AT: AT -> type(AT) ;
Inside_QUEST: QUEST -> type(QUEST) ;
Inside_ELVIS: ELVIS -> type(ELVIS) ;
Inside_LANGLE: LANGLE -> type(LANGLE) ;
Inside_RANGLE: RANGLE -> type(RANGLE) ;
Inside_LE: LE -> type(LE) ;
Inside_GE: GE -> type(GE) ;
Inside_EXCL_EQ: EXCL_EQ -> type(EXCL_EQ) ;
Inside_EXCL_EQEQ: EXCL_EQEQ -> type(EXCL_EQEQ) ;
Inside_NOT_IS: NOT_IS -> type(NOT_IS) ;
Inside_NOT_IN: NOT_IN -> type(NOT_IN) ;
Inside_AS_SAFE: AS_SAFE -> type(AS_SAFE) ;
Inside_EQEQ: EQEQ -> type(EQEQ) ;
Inside_EQEQEQ: EQEQEQ -> type(EQEQEQ) ;
Inside_SINGLE_QUOTE: SINGLE_QUOTE -> type(SINGLE_QUOTE) ;
Inside_QUOTE_OPEN: QUOTE_OPEN -> pushMode(LineString), type(QUOTE_OPEN) ;
Inside_TRIPLE_QUOTE_OPEN: TRIPLE_QUOTE_OPEN -> pushMode(MultiLineString), type(TRIPLE_QUOTE_OPEN) ;
Inside_VAL: VAL -> type(VAL) ;
Inside_VAR: VAR -> type(VAR) ;
Inside_OBJECT: OBJECT -> type(OBJECT) ;
Inside_SUPER: SUPER -> type(SUPER) ;
Inside_IN: IN -> type(IN) ;
Inside_OUT: OUT -> type(OUT) ;
Inside_FIELD: FIELD -> type(FIELD) ;
Inside_FILE: FILE -> type(FILE) ;
Inside_PROPERTY: PROPERTY -> type(PROPERTY) ;
Inside_GET: GET -> type(GET) ;
Inside_SET: SET -> type(SET) ;
Inside_RECEIVER: RECEIVER -> type(RECEIVER) ;
Inside_PARAM: PARAM -> type(PARAM) ;
Inside_SETPARAM: SETPARAM -> type(SETPARAM) ;
Inside_DELEGATE: DELEGATE -> type(DELEGATE) ;
Inside_THROW: THROW -> type(THROW) ;
Inside_RETURN: RETURN -> type(RETURN) ;
Inside_CONTINUE: CONTINUE -> type(CONTINUE) ;
Inside_BREAK: BREAK -> type(BREAK) ;
Inside_RETURN_AT: RETURN_AT -> type(RETURN_AT) ;
Inside_CONTINUE_AT: CONTINUE_AT -> type(CONTINUE_AT) ;
Inside_BREAK_AT: BREAK_AT -> type(BREAK_AT) ;
Inside_IF: IF -> type(IF) ;
Inside_ELSE: ELSE -> type(ELSE) ;
Inside_WHEN: WHEN -> type(WHEN) ;
Inside_TRY: TRY -> type(TRY) ;
Inside_CATCH: CATCH -> type(CATCH) ;
Inside_FINALLY: FINALLY -> type(FINALLY) ;
Inside_FOR: FOR -> type(FOR) ;
Inside_DO: DO -> type(DO) ;
Inside_WHILE: WHILE -> type(WHILE) ;
Inside_PUBLIC: PUBLIC -> type(PUBLIC) ;
Inside_PRIVATE: PRIVATE -> type(PRIVATE) ;
Inside_PROTECTED: PROTECTED -> type(PROTECTED) ;
Inside_INTERNAL: INTERNAL -> type(INTERNAL) ;
Inside_ENUM: ENUM -> type(ENUM) ;
Inside_SEALED: SEALED -> type(SEALED) ;
Inside_ANNOTATION: ANNOTATION -> type(ANNOTATION) ;
Inside_DATA: DATA -> type(DATA) ;
Inside_INNER: INNER -> type(INNER) ;
Inside_TAILREC: TAILREC -> type(TAILREC) ;
Inside_OPERATOR: OPERATOR -> type(OPERATOR) ;
Inside_INLINE: INLINE -> type(INLINE) ;
Inside_INFIX: INFIX -> type(INFIX) ;
Inside_EXTERNAL: EXTERNAL -> type(EXTERNAL) ;
Inside_SUSPEND: SUSPEND -> type(SUSPEND) ;
Inside_OVERRIDE: OVERRIDE -> type(OVERRIDE) ;
Inside_ABSTRACT: ABSTRACT -> type(ABSTRACT) ;
Inside_FINAL: FINAL -> type(FINAL) ;
Inside_OPEN: OPEN -> type(OPEN) ;
Inside_CONST: CONST -> type(CONST) ;
Inside_LATEINIT: LATEINIT -> type(LATEINIT) ;
Inside_VARARG: VARARG -> type(VARARG) ;
Inside_NOINLINE: NOINLINE -> type(NOINLINE) ;
Inside_CROSSINLINE: CROSSINLINE -> type(CROSSINLINE) ;
Inside_REIFIED: REIFIED -> type(REIFIED) ;
Inside_BooleanLiteral: BooleanLiteral -> type(BooleanLiteral) ;
Inside_IntegerLiteral: IntegerLiteral -> type(IntegerLiteral) ;
Inside_HexLiteral: HexLiteral -> type(HexLiteral) ;
Inside_BinLiteral: BinLiteral -> type(BinLiteral) ;
Inside_CharacterLiteral: CharacterLiteral -> type(CharacterLiteral) ;
Inside_RealLiteral: RealLiteral -> type(RealLiteral) ;
Inside_NullLiteral: NullLiteral -> type(NullLiteral) ;
Inside_LongLiteral: LongLiteral -> type(LongLiteral) ;
Inside_Identifier: Identifier -> type(Identifier) ;
Inside_LabelReference: LabelReference -> type(LabelReference) ;
Inside_LabelDefinition: LabelDefinition -> type(LabelDefinition) ;
Inside_Comment: (LineComment | DelimitedComment) -> channel(HIDDEN) ;
Inside_WS: WS -> skip ;
Inside_NL: NL -> skip ;
mode LineString ;
QUOTE_CLOSE
: '"' -> popMode
;
LineStrRef
: FieldIdentifier
;
LineStrText
: ~('\\' | '"' | '$')+ | '$'
;
LineStrEscapedChar
: '\\' .
| UniCharacterLiteral
;
LineStrExprStart
: '${' -> pushMode(StringExpression)
;
mode MultiLineString ;
TRIPLE_QUOTE_CLOSE
: MultiLineStringQuote? '"""' -> popMode
;
MultiLineStringQuote
: '"'+
;
MultiLineStrRef
: FieldIdentifier
;
MultiLineStrText
: ~('\\' | '"' | '$')+ | '$'
;
MultiLineStrEscapedChar
: '\\' .
;
MultiLineStrExprStart
: '${' -> pushMode(StringExpression)
;
MultiLineNL: NL -> skip ;
mode StringExpression ;
StrExpr_RCURL: RCURL -> popMode, type(RCURL) ;
StrExpr_LPAREN: LPAREN -> pushMode(Inside), type(LPAREN) ;
StrExpr_LSQUARE: LSQUARE -> pushMode(Inside), type(LSQUARE) ;
StrExpr_RPAREN: ')' -> type(RPAREN) ;
StrExpr_RSQUARE: ']' -> type(RSQUARE);
StrExpr_LCURL: LCURL -> pushMode(StringExpression), type(LCURL) ;
StrExpr_DOT: DOT -> type(DOT) ;
StrExpr_COMMA: COMMA -> type(COMMA) ;
StrExpr_MULT: MULT -> type(MULT) ;
StrExpr_MOD: MOD -> type(MOD) ;
StrExpr_DIV: DIV -> type(DIV) ;
StrExpr_ADD: ADD -> type(ADD) ;
StrExpr_SUB: SUB -> type(SUB) ;
StrExpr_INCR: INCR -> type(INCR) ;
StrExpr_DECR: DECR -> type(DECR) ;
StrExpr_CONJ: CONJ -> type(CONJ) ;
StrExpr_DISJ: DISJ -> type(DISJ) ;
StrExpr_EXCL: EXCL -> type(EXCL) ;
StrExpr_COLON: COLON -> type(COLON) ;
StrExpr_SEMICOLON: SEMICOLON -> type(SEMICOLON) ;
StrExpr_ASSIGNMENT: ASSIGNMENT -> type(ASSIGNMENT) ;
StrExpr_ADD_ASSIGNMENT: ADD_ASSIGNMENT -> type(ADD_ASSIGNMENT) ;
StrExpr_SUB_ASSIGNMENT: SUB_ASSIGNMENT -> type(SUB_ASSIGNMENT) ;
StrExpr_MULT_ASSIGNMENT: MULT_ASSIGNMENT -> type(MULT_ASSIGNMENT) ;
StrExpr_DIV_ASSIGNMENT: DIV_ASSIGNMENT -> type(DIV_ASSIGNMENT) ;
StrExpr_MOD_ASSIGNMENT: MOD_ASSIGNMENT -> type(MOD_ASSIGNMENT) ;
StrExpr_ARROW: ARROW -> type(ARROW) ;
StrExpr_DOUBLE_ARROW: DOUBLE_ARROW -> type(DOUBLE_ARROW) ;
StrExpr_RANGE: RANGE -> type(RANGE) ;
StrExpr_COLONCOLON: COLONCOLON -> type(COLONCOLON) ;
StrExpr_Q_COLONCOLON: Q_COLONCOLON -> type(Q_COLONCOLON) ;
StrExpr_DOUBLE_SEMICOLON: DOUBLE_SEMICOLON -> type(DOUBLE_SEMICOLON) ;
StrExpr_HASH: HASH -> type(HASH) ;
StrExpr_AT: AT -> type(AT) ;
StrExpr_QUEST: QUEST -> type(QUEST) ;
StrExpr_ELVIS: ELVIS -> type(ELVIS) ;
StrExpr_LANGLE: LANGLE -> type(LANGLE) ;
StrExpr_RANGLE: RANGLE -> type(RANGLE) ;
StrExpr_LE: LE -> type(LE) ;
StrExpr_GE: GE -> type(GE) ;
StrExpr_EXCL_EQ: EXCL_EQ -> type(EXCL_EQ) ;
StrExpr_EXCL_EQEQ: EXCL_EQEQ -> type(EXCL_EQEQ) ;
StrExpr_AS: AS -> type(IS) ;
StrExpr_IS: IS -> type(IN) ;
StrExpr_IN: IN ;
StrExpr_NOT_IS: NOT_IS -> type(NOT_IS) ;
StrExpr_NOT_IN: NOT_IN -> type(NOT_IN) ;
StrExpr_AS_SAFE: AS_SAFE -> type(AS_SAFE) ;
StrExpr_EQEQ: EQEQ -> type(EQEQ) ;
StrExpr_EQEQEQ: EQEQEQ -> type(EQEQEQ) ;
StrExpr_SINGLE_QUOTE: SINGLE_QUOTE -> type(SINGLE_QUOTE) ;
StrExpr_QUOTE_OPEN: QUOTE_OPEN -> pushMode(LineString), type(QUOTE_OPEN) ;
StrExpr_TRIPLE_QUOTE_OPEN: TRIPLE_QUOTE_OPEN -> pushMode(MultiLineString), type(TRIPLE_QUOTE_OPEN) ;
StrExpr_BooleanLiteral: BooleanLiteral -> type(BooleanLiteral) ;
StrExpr_IntegerLiteral: IntegerLiteral -> type(IntegerLiteral) ;
StrExpr_HexLiteral: HexLiteral -> type(HexLiteral) ;
StrExpr_BinLiteral: BinLiteral -> type(BinLiteral) ;
StrExpr_CharacterLiteral: CharacterLiteral -> type(CharacterLiteral) ;
StrExpr_RealLiteral: RealLiteral -> type(RealLiteral) ;
StrExpr_NullLiteral: NullLiteral -> type(NullLiteral) ;
StrExpr_LongLiteral: LongLiteral -> type(LongLiteral) ;
StrExpr_Identifier: Identifier -> type(Identifier) ;
StrExpr_LabelReference: LabelReference -> type(LabelReference) ;
StrExpr_LabelDefinition: LabelDefinition -> type(LabelDefinition) ;
StrExpr_Comment: (LineComment | DelimitedComment) -> channel(HIDDEN) ;
StrExpr_WS: WS -> skip ;
StrExpr_NL: NL -> skip ;

796
codegpt-core/src/main/antlr/KotlinParser.g4
View file

@ -1,796 +0,0 @@
/**
* Kotlin Grammar for ANTLR v4
*
* Based on:
* http://jetbrains.github.io/kotlin-spec/#_grammars_and_parsing
* and
* http://kotlinlang.org/docs/reference/grammar.html
*
* Tested on
* https://github.com/JetBrains/kotlin/tree/master/compiler/testData/psi
*/
parser grammar KotlinParser;
options { tokenVocab = KotlinLexer; }
kotlinFile
: NL* preamble anysemi* (topLevelObject (anysemi+ topLevelObject?)*)? EOF
;
script
: NL* preamble anysemi* (expression (anysemi+ expression?)*)? EOF
;
preamble
: fileAnnotations? packageHeader importList
;
fileAnnotations
: fileAnnotation+
;
fileAnnotation
: (FILE COLON (LSQUARE unescapedAnnotation+ RSQUARE | unescapedAnnotation) semi?)+
;
packageHeader
: (modifierList? PACKAGE identifier semi?)?
;
importList
: importHeader*
;
importHeader
: IMPORT identifier (DOT MULT | importAlias)? semi?
;
importAlias
: AS simpleIdentifier
;
topLevelObject
: classDeclaration
| objectDeclaration
| functionDeclaration
| propertyDeclaration
| typeAlias
;
classDeclaration
: modifierList? (CLASS | INTERFACE) NL* simpleIdentifier
(NL* typeParameters)? (NL* primaryConstructor)?
(NL* COLON NL* delegationSpecifiers)?
(NL* typeConstraints)?
(NL* classBody | NL* enumClassBody)?
;
primaryConstructor
: modifierList? (CONSTRUCTOR NL*)? classParameters
;
classParameters
: LPAREN (classParameter (COMMA classParameter)* COMMA?)? RPAREN
;
classParameter
: modifierList? (VAL | VAR)? simpleIdentifier COLON type (ASSIGNMENT expression)?
;
delegationSpecifiers
: annotations* delegationSpecifier (NL* COMMA NL* annotations* delegationSpecifier)*
;
delegationSpecifier
: constructorInvocation
| userType
| explicitDelegation
;
constructorInvocation
: userType callSuffix
;
explicitDelegation
: userType NL* BY NL* expression
;
classBody
: LCURL NL* classMemberDeclaration* NL* RCURL
;
classMemberDeclaration
: (classDeclaration
| functionDeclaration
| objectDeclaration
| companionObject
| propertyDeclaration
| anonymousInitializer
| secondaryConstructor
| typeAlias) anysemi+
;
anonymousInitializer
: INIT NL* block
;
secondaryConstructor
: modifierList? CONSTRUCTOR NL* functionValueParameters (NL* COLON NL* constructorDelegationCall)? NL* block?
;
constructorDelegationCall
: THIS NL* valueArguments
| SUPER NL* valueArguments
;
enumClassBody
: LCURL NL* enumEntries? (NL* SEMICOLON NL* classMemberDeclaration*)? NL* RCURL
;
enumEntries
: (enumEntry NL*)+ SEMICOLON?
;
enumEntry
: annotations* simpleIdentifier (NL* valueArguments)? (NL* classBody)? (NL* COMMA)?
;
functionDeclaration
: modifierList? FUN
(NL* type NL* DOT)?
(NL* typeParameters)?
(NL* receiverType NL* DOT)?
(NL* identifier)?
NL* functionValueParameters
(NL* COLON NL* type)?
(NL* typeConstraints)?
(NL* functionBody)?
;
functionValueParameters
: LPAREN (functionValueParameter (COMMA functionValueParameter)* COMMA?)? RPAREN
;
functionValueParameter
: modifierList? parameter (ASSIGNMENT expression)?
;
parameter
: simpleIdentifier COLON type
;
receiverType
: typeModifierList? (parenthesizedType | nullableType | typeReference)
;
functionBody
: block
| ASSIGNMENT NL* expression
;
objectDeclaration
: modifierList? OBJECT
NL* simpleIdentifier
(NL* primaryConstructor)?
(NL* COLON NL* delegationSpecifiers)?
(NL* classBody)?
;
companionObject
: modifierList? COMPANION NL* modifierList? OBJECT
(NL* simpleIdentifier)?
(NL* COLON NL* delegationSpecifiers)?
(NL* classBody)?
;
propertyDeclaration
: modifierList? (VAL | VAR)
(NL* typeParameters)?
(NL* type NL* DOT)?
(NL* (multiVariableDeclaration | variableDeclaration))
(NL* typeConstraints)?
(NL* (BY | ASSIGNMENT) NL* expression)?
(NL* getter (semi setter)? | NL* setter (semi getter)?)?
;
multiVariableDeclaration
: LPAREN variableDeclaration (COMMA variableDeclaration)* RPAREN
;
variableDeclaration
: simpleIdentifier (COLON type)?
;
getter
: modifierList? GETTER
| modifierList? GETTER NL* LPAREN RPAREN (NL* COLON NL* type)? NL* (block | ASSIGNMENT NL* expression)
;
setter
: modifierList? SETTER
| modifierList? SETTER NL* LPAREN (annotations | parameterModifier)* (simpleIdentifier | parameter) RPAREN NL* functionBody
;
typeAlias
: modifierList? TYPE_ALIAS NL* simpleIdentifier (NL* typeParameters)? NL* ASSIGNMENT NL* type
;
typeParameters
: LANGLE NL* typeParameter (NL* COMMA NL* typeParameter)* (NL* COMMA)? NL* RANGLE
;
typeParameter
: modifierList? NL* (simpleIdentifier | MULT) (NL* COLON NL* type)?
;
type
: typeModifierList?
( functionType
| parenthesizedType
| nullableType
| typeReference)
;
typeModifierList
: (annotations | SUSPEND NL*)+
;
parenthesizedType
: LPAREN type RPAREN
;
nullableType
: (typeReference | parenthesizedType) NL* QUEST+
;
typeReference
: LPAREN typeReference RPAREN
| userType
| DYNAMIC
;
functionType
: (functionTypeReceiver NL* DOT NL*)? functionTypeParameters NL* ARROW (NL* type)
;
functionTypeReceiver
: parenthesizedType
| nullableType
| typeReference
;
userType
: simpleUserType (NL* DOT NL* simpleUserType)*
;
simpleUserType
: simpleIdentifier (NL* typeArguments)?
;
//parameters for functionType
functionTypeParameters
: LPAREN NL* (parameter | type)? (NL* COMMA NL* (parameter | type))* (NL* COMMA)? NL* RPAREN
;
typeConstraints
: WHERE NL* typeConstraint (NL* COMMA NL* typeConstraint)*
;
typeConstraint
: annotations* simpleIdentifier NL* COLON NL* type
;
block
: LCURL statements RCURL
;
statements
: anysemi* (statement (anysemi+ statement?)*)?
;
statement
: declaration
| blockLevelExpression
;
blockLevelExpression
: annotations* NL* expression
;
declaration
: labelDefinition*
( classDeclaration
| functionDeclaration
| propertyDeclaration
| typeAlias)
;
expression
: disjunction (assignmentOperator disjunction)*
;
disjunction
: conjunction (NL* DISJ NL* conjunction)*
;
conjunction
: equalityComparison (NL* CONJ NL* equalityComparison)*
;
equalityComparison
: comparison (equalityOperation NL* comparison)*
;
comparison
: namedInfix (comparisonOperator NL* namedInfix)?
;
namedInfix
: elvisExpression ((inOperator NL* elvisExpression)+ | (isOperator NL* type))?
;
elvisExpression
: infixFunctionCall (NL* ELVIS NL* infixFunctionCall)*
;
infixFunctionCall
: rangeExpression (simpleIdentifier NL* rangeExpression)*
;
rangeExpression
: additiveExpression (RANGE NL* additiveExpression)*
;
additiveExpression
: multiplicativeExpression (additiveOperator NL* multiplicativeExpression)*
;
multiplicativeExpression
: typeRHS (multiplicativeOperation NL* typeRHS)*
;
typeRHS
: prefixUnaryExpression (NL* typeOperation prefixUnaryExpression)*
;
prefixUnaryExpression
: prefixUnaryOperation* postfixUnaryExpression
;
postfixUnaryExpression
: (atomicExpression | callableReference) postfixUnaryOperation*
;
atomicExpression
: parenthesizedExpression
| literalConstant
| functionLiteral
| thisExpression // THIS labelReference?
| superExpression // SUPER (LANGLE type RANGLE)? labelReference?
| conditionalExpression // ifExpression, whenExpression
| tryExpression
| objectLiteral
| jumpExpression
| loopExpression
| collectionLiteral
| simpleIdentifier
| VAL identifier
;
parenthesizedExpression
: LPAREN expression RPAREN
;
callSuffix
: typeArguments valueArguments? annotatedLambda*
| valueArguments annotatedLambda*
| annotatedLambda+
;
annotatedLambda
: unescapedAnnotation* LabelDefinition? NL* functionLiteral
;
arrayAccess
: LSQUARE (expression (COMMA expression)*)? RSQUARE
;
valueArguments
: LPAREN (valueArgument (COMMA valueArgument)* (NL* COMMA)?)? RPAREN
;
typeArguments
: LANGLE NL* typeProjection (NL* COMMA typeProjection)* (NL* COMMA)? NL* RANGLE QUEST?
;
typeProjection
: typeProjectionModifierList? type | MULT
;
typeProjectionModifierList
: varianceAnnotation+
;
valueArgument
: (simpleIdentifier NL* ASSIGNMENT NL*)? MULT? NL* expression
;
literalConstant
: BooleanLiteral
| IntegerLiteral
| stringLiteral
| HexLiteral
| BinLiteral
| CharacterLiteral
| RealLiteral
| NullLiteral
| LongLiteral
;
stringLiteral
: lineStringLiteral
| multiLineStringLiteral
;
lineStringLiteral
: QUOTE_OPEN (lineStringContent | lineStringExpression)* QUOTE_CLOSE
;
multiLineStringLiteral
: TRIPLE_QUOTE_OPEN (multiLineStringContent | multiLineStringExpression | lineStringLiteral | MultiLineStringQuote)* TRIPLE_QUOTE_CLOSE
;
lineStringContent
: LineStrText
| LineStrEscapedChar
| LineStrRef
;
lineStringExpression
: LineStrExprStart expression RCURL
;
multiLineStringContent
: MultiLineStrText
| MultiLineStrEscapedChar
| MultiLineStrRef
;
multiLineStringExpression
: MultiLineStrExprStart expression RCURL
;
functionLiteral
: annotations*
( LCURL NL* statements NL* RCURL
| LCURL NL* lambdaParameters NL* ARROW NL* statements NL* RCURL )
;
lambdaParameters
: lambdaParameter? (NL* COMMA NL* lambdaParameter)*
;
lambdaParameter
: variableDeclaration
| multiVariableDeclaration (NL* COLON NL* type)?
;
// https://kotlinlang.org/docs/reference/grammar.html#objectLiteral
objectLiteral
: OBJECT (NL* COLON NL* delegationSpecifiers)? NL* classBody?
;
collectionLiteral
: LSQUARE expression? (COMMA expression)* RSQUARE
;
thisExpression
: THIS LabelReference?
;
superExpression
: SUPER (LANGLE NL* type NL* RANGLE)? LabelReference?
;
conditionalExpression
: ifExpression
| whenExpression
;
ifExpression
: IF NL* LPAREN expression RPAREN NL* controlStructureBody? SEMICOLON?
(NL* ELSE NL* controlStructureBody?)?
;
controlStructureBody
: block
| expression
;
whenExpression
: WHEN NL* (LPAREN expression RPAREN)? NL* LCURL NL* (whenEntry NL*)* NL* RCURL
;
whenEntry
: whenCondition (NL* COMMA NL* whenCondition)* NL* ARROW NL* controlStructureBody semi?
| ELSE NL* ARROW NL* controlStructureBody
;
whenCondition
: expression
| rangeTest
| typeTest
;
rangeTest
: inOperator NL* expression
;
typeTest
: isOperator NL* type
;
tryExpression
: TRY NL* block (NL* catchBlock)* (NL* finallyBlock)?
;
catchBlock
: CATCH NL* LPAREN annotations* simpleIdentifier COLON userType RPAREN NL* block
;
finallyBlock
: FINALLY NL* block
;
loopExpression
: forExpression
| whileExpression
| doWhileExpression
;
forExpression
: FOR NL* LPAREN annotations* (variableDeclaration | multiVariableDeclaration) IN expression RPAREN NL* controlStructureBody?
;
whileExpression
: WHILE NL* LPAREN expression RPAREN NL* controlStructureBody?
;
doWhileExpression
: DO NL* controlStructureBody? NL* WHILE NL* LPAREN expression RPAREN
;
jumpExpression
: THROW NL* expression
| (RETURN | RETURN_AT) expression?
| CONTINUE | CONTINUE_AT
| BREAK | BREAK_AT
;
callableReference
: (userType (QUEST NL*)*)? NL* (COLONCOLON | Q_COLONCOLON) NL* (identifier | CLASS)
| THIS NL* COLONCOLON NL* CLASS
;
assignmentOperator
: ASSIGNMENT
| ADD_ASSIGNMENT
| SUB_ASSIGNMENT
| MULT_ASSIGNMENT
| DIV_ASSIGNMENT
| MOD_ASSIGNMENT
;
equalityOperation
: EXCL_EQ
| EXCL_EQEQ
| EQEQ
| EQEQEQ
;
comparisonOperator
: LANGLE
| RANGLE
| LE
| GE
;
inOperator
: IN | NOT_IN
;
isOperator
: IS | NOT_IS
;
additiveOperator
: ADD | SUB
;
multiplicativeOperation
: MULT
| DIV
| MOD
;
typeOperation
: AS
| AS_SAFE
| COLON
;
prefixUnaryOperation
: INCR
| DECR
| ADD
| SUB
| EXCL
| annotations
| labelDefinition
;
postfixUnaryOperation
: INCR | DECR | EXCL EXCL
| callSuffix
| arrayAccess
| NL* memberAccessOperator postfixUnaryExpression
;
memberAccessOperator
: DOT | QUEST DOT
;
modifierList
: (annotations | modifier)+
;
modifier
: (classModifier
| memberModifier
| visibilityModifier
| varianceAnnotation
| functionModifier
| propertyModifier
| inheritanceModifier
| parameterModifier
| typeParameterModifier) NL*
;
classModifier
: ENUM
| SEALED
| ANNOTATION
| DATA
| INNER
;
memberModifier
: OVERRIDE
| LATEINIT
;
visibilityModifier
: PUBLIC
| PRIVATE
| INTERNAL
| PROTECTED
;
varianceAnnotation
: IN | OUT
;
functionModifier
: TAILREC
| OPERATOR
| INFIX
| INLINE
| EXTERNAL
| SUSPEND
;
propertyModifier
: CONST
;
inheritanceModifier
: ABSTRACT
| FINAL
| OPEN
;
parameterModifier
: VARARG
| NOINLINE
| CROSSINLINE
;
typeParameterModifier
: REIFIED
;
labelDefinition
: LabelDefinition NL*
;
annotations
: (annotation | annotationList) NL*
;
annotation
: annotationUseSiteTarget NL* COLON NL* unescapedAnnotation
| LabelReference (NL* DOT NL* simpleIdentifier)* (NL* typeArguments)? (NL* valueArguments)?
;
annotationList
: annotationUseSiteTarget COLON LSQUARE unescapedAnnotation+ RSQUARE
| AT LSQUARE unescapedAnnotation+ RSQUARE
;
annotationUseSiteTarget
: FIELD
| FILE
| PROPERTY
| GET
| SET
| RECEIVER
| PARAM
| SETPARAM
| DELEGATE
;
unescapedAnnotation
: identifier typeArguments? valueArguments?
;
identifier
: simpleIdentifier (NL* DOT simpleIdentifier)*
;
simpleIdentifier
: Identifier
//soft keywords:
| ABSTRACT
| ANNOTATION
| BY
| CATCH
| COMPANION
| CONSTRUCTOR
| CROSSINLINE
| DATA
| DYNAMIC
| ENUM
| EXTERNAL
| FINAL
| FINALLY
| GETTER
| IMPORT
| INFIX
| INIT
| INLINE
| INNER
| INTERNAL
| LATEINIT
| NOINLINE
| OPEN
| OPERATOR
| OUT
| OVERRIDE
| PRIVATE
| PROTECTED
| PUBLIC
| REIFIED
| SEALED
| TAILREC
| SETTER
| VARARG
| WHERE
//strong keywords
| CONST
| SUSPEND
;
semi: NL+ | NL* SEMICOLON NL*;
anysemi: NL | SEMICOLON;

759
codegpt-core/src/main/antlr/PythonLexer.g4
View file

@ -1,759 +0,0 @@
/*
Python grammar.
The MIT License (MIT).
Copyright (c) 2014, Bart Kiers, bart@big-o.nl
Copyright (c) 2019, Dmitriy Litovchenko, Dmitry.Litovchenko1@yandex.ru, Positive Technologies
Copyright (c) 2019, Nikita Subbotin, sub.nik.and@gmail.com, Positive Technologies
Copyright (c) 2019, Ivan Kochurkin, kvanttt@gmail.com, Positive Technologies
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
lexer grammar PythonLexer;
options { superClass=PythonLexerBase; }
// Insert here @header for C++ lexer.
// Artificial tokens only for parser purposes
tokens { INDENT, DEDENT, LINE_BREAK }
// Keywords
DEF : 'def';
RETURN : 'return';
RAISE : 'raise';
FROM : 'from';
IMPORT : 'import';
NONLOCAL : 'nonlocal';
AS : 'as';
GLOBAL : 'global';
ASSERT : 'assert';
IF : 'if';
ELIF : 'elif';
ELSE : 'else';
WHILE : 'while';
FOR : 'for';
IN : 'in';
TRY : 'try';
NONE : 'None';
FINALLY : 'finally';
WITH : 'with';
EXCEPT : 'except';
LAMBDA : 'lambda';
OR : 'or';
AND : 'and';
NOT : 'not';
IS : 'is';
CLASS : 'class';
YIELD : 'yield';
DEL : 'del';
PASS : 'pass';
CONTINUE : 'continue';
BREAK : 'break';
ASYNC : 'async';
AWAIT : 'await';
PRINT : 'print';
EXEC : 'exec';
TRUE : 'True';
FALSE : 'False';
// Operators
DOT : '.';
ELLIPSIS : '...';
REVERSE_QUOTE : '`';
STAR : '*';
COMMA : ',';
COLON : ':';
SEMI_COLON : ';';
POWER : '**';
ASSIGN : '=';
OR_OP : '|';
XOR : '^';
AND_OP : '&';
LEFT_SHIFT : '<<';
RIGHT_SHIFT : '>>';
ADD : '+';
MINUS : '-';
DIV : '/';
MOD : '%';
IDIV : '//';
NOT_OP : '~';
LESS_THAN : '<';
GREATER_THAN : '>';
EQUALS : '==';
GT_EQ : '>=';
LT_EQ : '<=';
NOT_EQ_1 : '<>';
NOT_EQ_2 : '!=';
AT : '@';
ARROW : '->';
ADD_ASSIGN : '+=';
SUB_ASSIGN : '-=';
MULT_ASSIGN : '*=';
AT_ASSIGN : '@=';
DIV_ASSIGN : '/=';
MOD_ASSIGN : '%=';
AND_ASSIGN : '&=';
OR_ASSIGN : '|=';
XOR_ASSIGN : '^=';
LEFT_SHIFT_ASSIGN : '<<=';
RIGHT_SHIFT_ASSIGN : '>>=';
POWER_ASSIGN : '**=';
IDIV_ASSIGN : '//=';
STRING : ([uU] | [fF] [rR]? | [rR] [fF]?)? (SHORT_STRING | LONG_STRING)
| ([bB] [rR]? | [rR] [bB]) (SHORT_BYTES | LONG_BYTES)
;
DECIMAL_INTEGER : [1-9] [0-9]*
| '0'+
;
OCT_INTEGER : '0' [oO] [0-7]+;
HEX_INTEGER : '0' [xX] [0-9a-fA-F]+;
BIN_INTEGER : '0' [bB] [01]+;
IMAG_NUMBER : (EXPONENT_OR_POINT_FLOAT | [0-9]+) [jJ];
FLOAT_NUMBER : EXPONENT_OR_POINT_FLOAT;
OPEN_PAREN : '(' {this.IncIndentLevel();};
CLOSE_PAREN : ')' {this.DecIndentLevel();};
OPEN_BRACE : '{' {this.IncIndentLevel();};
CLOSE_BRACE : '}' {this.DecIndentLevel();};
OPEN_BRACKET : '[' {this.IncIndentLevel();};
CLOSE_BRACKET : ']' {this.DecIndentLevel();};
NAME : ID_START ID_CONTINUE*;
LINE_JOIN : '\\' [ \t]* RN -> channel(HIDDEN);
NEWLINE : RN {this.HandleNewLine();} -> channel(HIDDEN);
WS : [ \t]+ {this.HandleSpaces();} -> channel(HIDDEN);
COMMENT : '#' ~[\r\n\f]* -> channel(HIDDEN);
// Fragments
fragment SHORT_STRING
: '\'' ('\\' (RN | .) | ~[\\\r\n'])* '\''
| '"' ('\\' (RN | .) | ~[\\\r\n"])* '"'
;
fragment LONG_STRING
: '\'\'\'' LONG_STRING_ITEM*? '\'\'\''
| '"""' LONG_STRING_ITEM*? '"""'
;
fragment LONG_STRING_ITEM
: ~'\\'
| '\\' (RN | .)
;
fragment RN
: '\r'? '\n'
;
fragment EXPONENT_OR_POINT_FLOAT
: ([0-9]+ | POINT_FLOAT) [eE] [+-]? [0-9]+
| POINT_FLOAT
;
fragment POINT_FLOAT
: [0-9]* '.' [0-9]+
| [0-9]+ '.'
;
fragment SHORT_BYTES
: '\'' (SHORT_BYTES_CHAR_NO_SINGLE_QUOTE | BYTES_ESCAPE_SEQ)* '\''
| '"' (SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE | BYTES_ESCAPE_SEQ)* '"'
;
fragment LONG_BYTES
: '\'\'\'' LONG_BYTES_ITEM*? '\'\'\''
| '"""' LONG_BYTES_ITEM*? '"""'
;
fragment LONG_BYTES_ITEM
: LONG_BYTES_CHAR
| BYTES_ESCAPE_SEQ
;
fragment SHORT_BYTES_CHAR_NO_SINGLE_QUOTE
: [\u0000-\u0009]
| [\u000B-\u000C]
| [\u000E-\u0026]
| [\u0028-\u005B]
| [\u005D-\u007F]
;
fragment SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE
: [\u0000-\u0009]
| [\u000B-\u000C]
| [\u000E-\u0021]
| [\u0023-\u005B]
| [\u005D-\u007F]
;
/// Any ASCII character except "\"
fragment LONG_BYTES_CHAR
: [\u0000-\u005B]
| [\u005D-\u007F]
;
/// "\" <any ASCII character>
fragment BYTES_ESCAPE_SEQ
: '\\' [\u0000-\u007F]
;
/// All characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property
fragment ID_CONTINUE
: ID_START
| [0-9]
| [\u0300-\u036F]
| [\u0483-\u0486]
| [\u0591-\u05B9]
| [\u05BB-\u05BD]
| '\u05BF'
| [\u05C1-\u05C2]
| [\u05C4-\u05C5]
| '\u05C7'
| [\u0610-\u0615]
| [\u064B-\u065E]
| [\u0660-\u0669]
| '\u0670'
| [\u06D6-\u06DC]
| [\u06DF-\u06E4]
| [\u06E7-\u06E8]
| [\u06EA-\u06ED]
| [\u06F0-\u06F9]
| '\u0711'
| [\u0730-\u074A]
| [\u07A6-\u07B0]
| [\u0901-\u0902]
| '\u0903'
| '\u093C'
| [\u093E-\u0940]
| [\u0941-\u0948]
| [\u0949-\u094C]
| '\u094D'
| [\u0951-\u0954]
| [\u0962-\u0963]
| [\u0966-\u096F]
| '\u0981'
| [\u0982-\u0983]
| '\u09BC'
| [\u09BE-\u09C0]
| [\u09C1-\u09C4]
| [\u09C7-\u09C8]
| [\u09CB-\u09CC]
| '\u09CD'
| '\u09D7'
| [\u09E2-\u09E3]
| [\u09E6-\u09EF]
| [\u0A01-\u0A02]
| '\u0A03'
| '\u0A3C'
| [\u0A3E-\u0A40]
| [\u0A41-\u0A42]
| [\u0A47-\u0A48]
| [\u0A4B-\u0A4D]
| [\u0A66-\u0A6F]
| [\u0A70-\u0A71]
| [\u0A81-\u0A82]
| '\u0A83'
| '\u0ABC'
| [\u0ABE-\u0AC0]
| [\u0AC1-\u0AC5]
| [\u0AC7-\u0AC8]
| '\u0AC9'
| [\u0ACB-\u0ACC]
| '\u0ACD'
| [\u0AE2-\u0AE3]
| [\u0AE6-\u0AEF]
| '\u0B01'
| [\u0B02-\u0B03]
| '\u0B3C'
| '\u0B3E'
| '\u0B3F'
| '\u0B40'
| [\u0B41-\u0B43]
| [\u0B47-\u0B48]
| [\u0B4B-\u0B4C]
| '\u0B4D'
| '\u0B56'
| '\u0B57'
| [\u0B66-\u0B6F]
| '\u0B82'
| [\u0BBE-\u0BBF]
| '\u0BC0'
| [\u0BC1-\u0BC2]
| [\u0BC6-\u0BC8]
| [\u0BCA-\u0BCC]
| '\u0BCD'
| '\u0BD7'
| [\u0BE6-\u0BEF]
| [\u0C01-\u0C03]
| [\u0C3E-\u0C40]
| [\u0C41-\u0C44]
| [\u0C46-\u0C48]
| [\u0C4A-\u0C4D]
| [\u0C55-\u0C56]
| [\u0C66-\u0C6F]
| [\u0C82-\u0C83]
| '\u0CBC'
| '\u0CBE'
| '\u0CBF'
| [\u0CC0-\u0CC4]
| '\u0CC6'
| [\u0CC7-\u0CC8]
| [\u0CCA-\u0CCB]
| [\u0CCC-\u0CCD]
| [\u0CD5-\u0CD6]
| [\u0CE6-\u0CEF]
| [\u0D02-\u0D03]
| [\u0D3E-\u0D40]
| [\u0D41-\u0D43]
| [\u0D46-\u0D48]
| [\u0D4A-\u0D4C]
| '\u0D4D'
| '\u0D57'
| [\u0D66-\u0D6F]
| [\u0D82-\u0D83]
| '\u0DCA'
| [\u0DCF-\u0DD1]
| [\u0DD2-\u0DD4]
| '\u0DD6'
| [\u0DD8-\u0DDF]
| [\u0DF2-\u0DF3]
| '\u0E31'
| [\u0E34-\u0E3A]
| [\u0E47-\u0E4E]
| [\u0E50-\u0E59]
| '\u0EB1'
| [\u0EB4-\u0EB9]
| [\u0EBB-\u0EBC]
| [\u0EC8-\u0ECD]
| [\u0ED0-\u0ED9]
| [\u0F18-\u0F19]
| [\u0F20-\u0F29]
| '\u0F35'
| '\u0F37'
| '\u0F39'
| [\u0F3E-\u0F3F]
| [\u0F71-\u0F7E]
| '\u0F7F'
| [\u0F80-\u0F84]
| [\u0F86-\u0F87]
| [\u0F90-\u0F97]
| [\u0F99-\u0FBC]
| '\u0FC6'
| '\u102C'
| [\u102D-\u1030]
| '\u1031'
| '\u1032'
| [\u1036-\u1037]
| '\u1038'
| '\u1039'
| [\u1040-\u1049]
| [\u1056-\u1057]
| [\u1058-\u1059]
| '\u135F'
| [\u1369-\u1371]
| [\u1712-\u1714]
| [\u1732-\u1734]
| [\u1752-\u1753]
| [\u1772-\u1773]
| '\u17B6'
| [\u17B7-\u17BD]
| [\u17BE-\u17C5]
| '\u17C6'
| [\u17C7-\u17C8]
| [\u17C9-\u17D3]
| '\u17DD'
| [\u17E0-\u17E9]
| [\u180B-\u180D]
| [\u1810-\u1819]
| '\u18A9'
| [\u1920-\u1922]
| [\u1923-\u1926]
| [\u1927-\u1928]
| [\u1929-\u192B]
| [\u1930-\u1931]
| '\u1932'
| [\u1933-\u1938]
| [\u1939-\u193B]
| [\u1946-\u194F]
| [\u19B0-\u19C0]
| [\u19C8-\u19C9]
| [\u19D0-\u19D9]
| [\u1A17-\u1A18]
| [\u1A19-\u1A1B]
| [\u1DC0-\u1DC3]
| [\u203F-\u2040]
| '\u2054'
| [\u20D0-\u20DC]
| '\u20E1'
| [\u20E5-\u20EB]
| [\u302A-\u302F]
| [\u3099-\u309A]
| '\uA802'
| '\uA806'
| '\uA80B'
| [\uA823-\uA824]
| [\uA825-\uA826]
| '\uA827'
| '\uFB1E'
| [\uFE00-\uFE0F]
| [\uFE20-\uFE23]
| [\uFE33-\uFE34]
| [\uFE4D-\uFE4F]
| [\uFF10-\uFF19]
| '\uFF3F'
;
/// All characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property
fragment ID_START
: '_'
| [A-Z]
| [a-z]
| '\u00AA'
| '\u00B5'
| '\u00BA'
| [\u00C0-\u00D6]
| [\u00D8-\u00F6]
| [\u00F8-\u01BA]
| '\u01BB'
| [\u01BC-\u01BF]
| [\u01C0-\u01C3]
| [\u01C4-\u0241]
| [\u0250-\u02AF]
| [\u02B0-\u02C1]
| [\u02C6-\u02D1]
| [\u02E0-\u02E4]
| '\u02EE'
| '\u037A'
| '\u0386'
| [\u0388-\u038A]
| '\u038C'
| [\u038E-\u03A1]
| [\u03A3-\u03CE]
| [\u03D0-\u03F5]
| [\u03F7-\u0481]
| [\u048A-\u04CE]
| [\u04D0-\u04F9]
| [\u0500-\u050F]
| [\u0531-\u0556]
| '\u0559'
| [\u0561-\u0587]
| [\u05D0-\u05EA]
| [\u05F0-\u05F2]
| [\u0621-\u063A]
| '\u0640'
| [\u0641-\u064A]
| [\u066E-\u066F]
| [\u0671-\u06D3]
| '\u06D5'
| [\u06E5-\u06E6]
| [\u06EE-\u06EF]
| [\u06FA-\u06FC]
| '\u06FF'
| '\u0710'
| [\u0712-\u072F]
| [\u074D-\u076D]
| [\u0780-\u07A5]
| '\u07B1'
| [\u0904-\u0939]
| '\u093D'
| '\u0950'
| [\u0958-\u0961]
| '\u097D'
| [\u0985-\u098C]
| [\u098F-\u0990]
| [\u0993-\u09A8]
| [\u09AA-\u09B0]
| '\u09B2'
| [\u09B6-\u09B9]
| '\u09BD'
| '\u09CE'
| [\u09DC-\u09DD]
| [\u09DF-\u09E1]
| [\u09F0-\u09F1]
| [\u0A05-\u0A0A]
| [\u0A0F-\u0A10]
| [\u0A13-\u0A28]
| [\u0A2A-\u0A30]
| [\u0A32-\u0A33]
| [\u0A35-\u0A36]
| [\u0A38-\u0A39]
| [\u0A59-\u0A5C]
| '\u0A5E'
| [\u0A72-\u0A74]
| [\u0A85-\u0A8D]
| [\u0A8F-\u0A91]
| [\u0A93-\u0AA8]
| [\u0AAA-\u0AB0]
| [\u0AB2-\u0AB3]
| [\u0AB5-\u0AB9]
| '\u0ABD'
| '\u0AD0'
| [\u0AE0-\u0AE1]
| [\u0B05-\u0B0C]
| [\u0B0F-\u0B10]
| [\u0B13-\u0B28]
| [\u0B2A-\u0B30]
| [\u0B32-\u0B33]
| [\u0B35-\u0B39]
| '\u0B3D'
| [\u0B5C-\u0B5D]
| [\u0B5F-\u0B61]
| '\u0B71'
| '\u0B83'
| [\u0B85-\u0B8A]
| [\u0B8E-\u0B90]
| [\u0B92-\u0B95]
| [\u0B99-\u0B9A]
| '\u0B9C'
| [\u0B9E-\u0B9F]
| [\u0BA3-\u0BA4]
| [\u0BA8-\u0BAA]
| [\u0BAE-\u0BB9]
| [\u0C05-\u0C0C]
| [\u0C0E-\u0C10]
| [\u0C12-\u0C28]
| [\u0C2A-\u0C33]
| [\u0C35-\u0C39]
| [\u0C60-\u0C61]
| [\u0C85-\u0C8C]
| [\u0C8E-\u0C90]
| [\u0C92-\u0CA8]
| [\u0CAA-\u0CB3]
| [\u0CB5-\u0CB9]
| '\u0CBD'
| '\u0CDE'
| [\u0CE0-\u0CE1]
| [\u0D05-\u0D0C]
| [\u0D0E-\u0D10]
| [\u0D12-\u0D28]
| [\u0D2A-\u0D39]
| [\u0D60-\u0D61]
| [\u0D85-\u0D96]
| [\u0D9A-\u0DB1]
| [\u0DB3-\u0DBB]
| '\u0DBD'
| [\u0DC0-\u0DC6]
| [\u0E01-\u0E30]
| [\u0E32-\u0E33]
| [\u0E40-\u0E45]
| '\u0E46'
| [\u0E81-\u0E82]
| '\u0E84'
| [\u0E87-\u0E88]
| '\u0E8A'
| '\u0E8D'
| [\u0E94-\u0E97]
| [\u0E99-\u0E9F]
| [\u0EA1-\u0EA3]
| '\u0EA5'
| '\u0EA7'
| [\u0EAA-\u0EAB]
| [\u0EAD-\u0EB0]
| [\u0EB2-\u0EB3]
| '\u0EBD'
| [\u0EC0-\u0EC4]
| '\u0EC6'
| [\u0EDC-\u0EDD]
| '\u0F00'
| [\u0F40-\u0F47]
| [\u0F49-\u0F6A]
| [\u0F88-\u0F8B]
| [\u1000-\u1021]
| [\u1023-\u1027]
| [\u1029-\u102A]
| [\u1050-\u1055]
| [\u10A0-\u10C5]
| [\u10D0-\u10FA]
| '\u10FC'
| [\u1100-\u1159]
| [\u115F-\u11A2]
| [\u11A8-\u11F9]
| [\u1200-\u1248]
| [\u124A-\u124D]
| [\u1250-\u1256]
| '\u1258'
| [\u125A-\u125D]
| [\u1260-\u1288]
| [\u128A-\u128D]
| [\u1290-\u12B0]
| [\u12B2-\u12B5]
| [\u12B8-\u12BE]
| '\u12C0'
| [\u12C2-\u12C5]
| [\u12C8-\u12D6]
| [\u12D8-\u1310]
| [\u1312-\u1315]
| [\u1318-\u135A]
| [\u1380-\u138F]
| [\u13A0-\u13F4]
| [\u1401-\u166C]
| [\u166F-\u1676]
| [\u1681-\u169A]
| [\u16A0-\u16EA]
| [\u16EE-\u16F0]
| [\u1700-\u170C]
| [\u170E-\u1711]
| [\u1720-\u1731]
| [\u1740-\u1751]
| [\u1760-\u176C]
| [\u176E-\u1770]
| [\u1780-\u17B3]
| '\u17D7'
| '\u17DC'
| [\u1820-\u1842]
| '\u1843'
| [\u1844-\u1877]
| [\u1880-\u18A8]
| [\u1900-\u191C]
| [\u1950-\u196D]
| [\u1970-\u1974]
| [\u1980-\u19A9]
| [\u19C1-\u19C7]
| [\u1A00-\u1A16]
| [\u1D00-\u1D2B]
| [\u1D2C-\u1D61]
| [\u1D62-\u1D77]
| '\u1D78'
| [\u1D79-\u1D9A]
| [\u1D9B-\u1DBF]
| [\u1E00-\u1E9B]
| [\u1EA0-\u1EF9]
| [\u1F00-\u1F15]
| [\u1F18-\u1F1D]
| [\u1F20-\u1F45]
| [\u1F48-\u1F4D]
| [\u1F50-\u1F57]
| '\u1F59'
| '\u1F5B'
| '\u1F5D'
| [\u1F5F-\u1F7D]
| [\u1F80-\u1FB4]
| [\u1FB6-\u1FBC]
| '\u1FBE'
| [\u1FC2-\u1FC4]
| [\u1FC6-\u1FCC]
| [\u1FD0-\u1FD3]
| [\u1FD6-\u1FDB]
| [\u1FE0-\u1FEC]
| [\u1FF2-\u1FF4]
| [\u1FF6-\u1FFC]
| '\u2071'
| '\u207F'
| [\u2090-\u2094]
| '\u2102'
| '\u2107'
| [\u210A-\u2113]
| '\u2115'
| '\u2118'
| [\u2119-\u211D]
| '\u2124'
| '\u2126'
| '\u2128'
| [\u212A-\u212D]
| '\u212E'
| [\u212F-\u2131]
| [\u2133-\u2134]
| [\u2135-\u2138]
| '\u2139'
| [\u213C-\u213F]
| [\u2145-\u2149]
| [\u2160-\u2183]
| [\u2C00-\u2C2E]
| [\u2C30-\u2C5E]
| [\u2C80-\u2CE4]
| [\u2D00-\u2D25]
| [\u2D30-\u2D65]
| '\u2D6F'
| [\u2D80-\u2D96]
| [\u2DA0-\u2DA6]
| [\u2DA8-\u2DAE]
| [\u2DB0-\u2DB6]
| [\u2DB8-\u2DBE]
| [\u2DC0-\u2DC6]
| [\u2DC8-\u2DCE]
| [\u2DD0-\u2DD6]
| [\u2DD8-\u2DDE]
| '\u3005'
| '\u3006'
| '\u3007'
| [\u3021-\u3029]
| [\u3031-\u3035]
| [\u3038-\u303A]
| '\u303B'
| '\u303C'
| [\u3041-\u3096]
| [\u309B-\u309C]
| [\u309D-\u309E]
| '\u309F'
| [\u30A1-\u30FA]
| [\u30FC-\u30FE]
| '\u30FF'
| [\u3105-\u312C]
| [\u3131-\u318E]
| [\u31A0-\u31B7]
| [\u31F0-\u31FF]
| [\u3400-\u4DB5]
| [\u4E00-\u9FBB]
| [\uA000-\uA014]
| '\uA015'
| [\uA016-\uA48C]
| [\uA800-\uA801]
| [\uA803-\uA805]
| [\uA807-\uA80A]
| [\uA80C-\uA822]
| [\uAC00-\uD7A3]
| [\uF900-\uFA2D]
| [\uFA30-\uFA6A]
| [\uFA70-\uFAD9]
| [\uFB00-\uFB06]
| [\uFB13-\uFB17]
| '\uFB1D'
| [\uFB1F-\uFB28]
| [\uFB2A-\uFB36]
| [\uFB38-\uFB3C]
| '\uFB3E'
| [\uFB40-\uFB41]
| [\uFB43-\uFB44]
| [\uFB46-\uFBB1]
| [\uFBD3-\uFD3D]
| [\uFD50-\uFD8F]
| [\uFD92-\uFDC7]
| [\uFDF0-\uFDFB]
| [\uFE70-\uFE74]
| [\uFE76-\uFEFC]
| [\uFF21-\uFF3A]
| [\uFF41-\uFF5A]
| [\uFF66-\uFF6F]
| '\uFF70'
| [\uFF71-\uFF9D]
| [\uFF9E-\uFF9F]
| [\uFFA0-\uFFBE]
| [\uFFC2-\uFFC7]
| [\uFFCA-\uFFCF]
| [\uFFD2-\uFFD7]
| [\uFFDA-\uFFDC]
;

386
codegpt-core/src/main/antlr/PythonParser.g4
View file

@ -1,386 +0,0 @@
/*
Python grammar.
The MIT License (MIT).
Copyright (c) 2014, Bart Kiers, bart@big-o.nl
Copyright (c) 2019, Dmitriy Litovchenko, Dmitry.Litovchenko1@yandex.ru, Positive Technologies
Copyright (c) 2019, Nikita Subbotin, sub.nik.and@gmail.com, Positive Technologies
Copyright (c) 2019, Ivan Kochurkin, kvanttt@gmail.com, Positive Technologies
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
parser grammar PythonParser;
// Insert here @header for C++ parser.
options { tokenVocab=PythonLexer; superClass=PythonParserBase; }
root
: (single_input
| file_input
| eval_input)? EOF
;
// A single interactive statement;
single_input
: LINE_BREAK
| simple_stmt
| compound_stmt LINE_BREAK
;
// A module or sequence of commands read from an input file
file_input
: (LINE_BREAK | stmt)+
;
// An input for the eval() and input() functions
eval_input
: testlist LINE_BREAK*
;
stmt
: simple_stmt
| compound_stmt
;
compound_stmt
: IF cond=test COLON suite elif_clause* else_clause? #if_stmt
| WHILE test COLON suite else_clause? #while_stmt
| ASYNC? FOR exprlist IN testlist COLON suite else_clause? #for_stmt
| TRY COLON suite (except_clause+ else_clause? finally_clause? | finally_clause) #try_stmt
| ASYNC? WITH with_item (COMMA with_item)* COLON suite #with_stmt
| decorator* (classdef | funcdef) #class_or_func_def_stmt
;
suite
: simple_stmt
| LINE_BREAK INDENT stmt+ DEDENT
;
decorator
: AT dotted_name (OPEN_PAREN arglist? CLOSE_PAREN)? LINE_BREAK
;
elif_clause
: ELIF test COLON suite
;
else_clause
: ELSE COLON suite
;
finally_clause
: FINALLY COLON suite
;
with_item
// NB compile.c makes sure that the default except clause is last
: test (AS expr)?
;
// Python 2 : EXCEPT test COMMA name
// Python 3 : EXCEPT test AS name
except_clause
: EXCEPT (test ({this.CheckVersion(2)}? COMMA name {this.SetVersion(2);} | {this.CheckVersion(3)}? AS name {this.SetVersion(3);})?)? COLON suite
;
classdef
: CLASS name (OPEN_PAREN arglist? CLOSE_PAREN)? COLON suite
;
funcdef
: ASYNC? DEF name OPEN_PAREN typedargslist? CLOSE_PAREN (ARROW test)? COLON suite
;
// python 3 paramters
// parameters list may have a trailing comma
typedargslist
: (def_parameters COMMA)? (args (COMMA def_parameters)? (COMMA kwargs)? | kwargs) COMMA?
| def_parameters COMMA?
;
args
: STAR named_parameter
;
kwargs
: POWER named_parameter
;
def_parameters
: def_parameter (COMMA def_parameter)*
;
// TODO: bare STAR parameter must follow named ones
def_parameter
: named_parameter (ASSIGN test)?
| STAR
;
named_parameter
: name (COLON test)?
;
simple_stmt
: small_stmt (SEMI_COLON small_stmt)* SEMI_COLON? (LINE_BREAK | EOF)
;
// TODO 1: left part augmented assignment should be `test` only, no stars or lists
// TODO 2: semantically annotated declaration is not an assignment
small_stmt
: testlist_star_expr assign_part? #expr_stmt
| {this.CheckVersion(2)}? PRINT ((test (COMMA test)* COMMA?)
| RIGHT_SHIFT test ((COMMA test)+ COMMA?)) {this.SetVersion(2);} #print_stmt // Python 2
| DEL exprlist #del_stmt
| PASS #pass_stmt
| BREAK #break_stmt
| CONTINUE #continue_stmt
| RETURN testlist? #return_stmt
| RAISE (test (COMMA test (COMMA test)?)?)? (FROM test)? #raise_stmt
| yield_expr #yield_stmt
| IMPORT dotted_as_names #import_stmt
| FROM ((DOT | ELLIPSIS)* dotted_name | (DOT | ELLIPSIS)+)
IMPORT (STAR | OPEN_PAREN import_as_names CLOSE_PAREN | import_as_names) #from_stmt
| GLOBAL name (COMMA name)* #global_stmt
| {this.CheckVersion(2)}? EXEC expr (IN test (COMMA test)?)? {this.SetVersion(2);} #exec_stmt // Python 2
| ASSERT test (COMMA test)? #assert_stmt
| {this.CheckVersion(3)}? NONLOCAL name (COMMA name)* {this.SetVersion(3);} #nonlocal_stmt // Python 3
;
testlist_star_expr
: ((test | star_expr) COMMA)+ (test | star_expr)?
| testlist
;
star_expr
: STAR expr
;
assign_part
// if left expression in assign is bool literal, it's mean that is Python 2 here
: ASSIGN ( testlist_star_expr (ASSIGN testlist_star_expr)* (ASSIGN yield_expr)?
| yield_expr)
| {this.CheckVersion(3)}? COLON test (ASSIGN testlist)? {this.SetVersion(3);} // annassign Python3 rule
| op=( ADD_ASSIGN
| SUB_ASSIGN
| MULT_ASSIGN
| AT_ASSIGN
| DIV_ASSIGN
| MOD_ASSIGN
| AND_ASSIGN
| OR_ASSIGN
| XOR_ASSIGN
| LEFT_SHIFT_ASSIGN
| RIGHT_SHIFT_ASSIGN
| POWER_ASSIGN
| IDIV_ASSIGN
)
(yield_expr | testlist)
;
exprlist
: expr (COMMA expr)* COMMA?
;
import_as_names
: import_as_name (COMMA import_as_name)* COMMA?
;
// TODO: that means we can use keyword True as the name here: `from foo import bar as True` -- no
import_as_name
: name (AS name)?
;
dotted_as_names
: dotted_as_name (COMMA dotted_as_name)*
;
dotted_as_name
: dotted_name (AS name)?
;
/*
* Warning!
* According to https://docs.python.org/3/reference/expressions.html#lambda LAMBDA should be followed by
* `parameter_list` (in our case it is `typedargslist`)
* But that's not true! `typedargslist` may have parameters with type hinting, but that's not permitted in lambda
* definition
*/
// https://docs.python.org/3/reference/expressions.html#operator-precedence
test
: logical_test (IF logical_test ELSE test)?
| LAMBDA varargslist? COLON test
;
// the same as `typedargslist`, but with no types
varargslist
: (vardef_parameters COMMA)? (varargs (COMMA vardef_parameters)? (COMMA varkwargs)? | varkwargs) COMMA?
| vardef_parameters COMMA?
;
vardef_parameters
: vardef_parameter (COMMA vardef_parameter)*
;
// TODO: bare STAR parameter must follow named ones
vardef_parameter
: name (ASSIGN test)?
| STAR
;
varargs
: STAR name
;
varkwargs
: POWER name
;
logical_test
: comparison
| NOT logical_test
| logical_test op=AND logical_test
| logical_test op=OR logical_test
;
comparison
: comparison (LESS_THAN | GREATER_THAN | EQUALS | GT_EQ | LT_EQ | NOT_EQ_1 | NOT_EQ_2 | optional=NOT? IN | IS optional=NOT?) comparison
| expr
;
expr
: AWAIT? atom trailer*
| <assoc=right> expr op=POWER expr
| op=(ADD | MINUS | NOT_OP) expr
| expr op=(STAR | DIV | MOD | IDIV | AT) expr
| expr op=(ADD | MINUS) expr
| expr op=(LEFT_SHIFT | RIGHT_SHIFT) expr
| expr op=AND_OP expr
| expr op=XOR expr
| expr op=OR_OP expr
;
atom
: OPEN_PAREN (yield_expr | testlist_comp)? CLOSE_PAREN
| OPEN_BRACKET testlist_comp? CLOSE_BRACKET
| OPEN_BRACE dictorsetmaker? CLOSE_BRACE
| REVERSE_QUOTE testlist COMMA? REVERSE_QUOTE
| ELLIPSIS
| name
| PRINT
| EXEC
| MINUS? number
| NONE
| STRING+
;
dictorsetmaker
: (test COLON test | POWER expr) (COMMA (test COLON test | POWER expr))* COMMA? // key_datum_list
| test COLON test comp_for // dict_comprehension
| testlist_comp
;
testlist_comp
: (test | star_expr) (comp_for | (COMMA (test | star_expr))* COMMA?)
;
testlist
: test (COMMA test)* COMMA?
;
dotted_name
: dotted_name DOT name
| name
;
name
: NAME
| TRUE
| FALSE
;
number
: integer
| IMAG_NUMBER
| FLOAT_NUMBER
;
integer
: DECIMAL_INTEGER
| OCT_INTEGER
| HEX_INTEGER
| BIN_INTEGER
;
yield_expr
: YIELD yield_arg?
;
yield_arg
: FROM test
| testlist
;
// TODO: this way we can pass: `f(x for x in i, a)`, but it's invalid.
// See: https://docs.python.org/3/reference/expressions.html#calls
trailer
: DOT name arguments?
| arguments
;
arguments
: OPEN_PAREN arglist? CLOSE_PAREN
| OPEN_BRACKET subscriptlist CLOSE_BRACKET
;
arglist
// The reason that keywords are test nodes instead of name is that using name
// results in an ambiguity. ast.c makes sure it's a name.
: argument (COMMA argument)* COMMA?
;
argument
: test (comp_for | ASSIGN test)?
| (POWER | STAR) test
;
// TODO: maybe inline?
subscriptlist
: subscript (COMMA subscript)* COMMA?
;
subscript
: ELLIPSIS
| test (COLON test? sliceop?)?
| COLON test? sliceop?
;
// TODO: maybe inline?
sliceop
: COLON test?
;
comp_for
: FOR exprlist IN logical_test comp_iter?
;
comp_iter
: comp_for
| IF test comp_iter?
;

308
codegpt-core/src/main/antlr/TypeScriptLexer.g4
View file

@ -1,308 +0,0 @@
lexer grammar TypeScriptLexer;
channels { ERROR }
options {
superClass=TypeScriptLexerBase;
}
MultiLineComment: '/*' .*? '*/' -> channel(HIDDEN);
SingleLineComment: '//' ~[\r\n\u2028\u2029]* -> channel(HIDDEN);
RegularExpressionLiteral: '/' RegularExpressionFirstChar RegularExpressionChar* {this.IsRegexPossible()}? '/' IdentifierPart*;
OpenBracket: '[';
CloseBracket: ']';
OpenParen: '(';
CloseParen: ')';
OpenBrace: '{' {this.ProcessOpenBrace();};
TemplateCloseBrace: {this.IsInTemplateString()}? '}' -> popMode;
CloseBrace: '}' {this.ProcessCloseBrace();};
SemiColon: ';';
Comma: ',';
Assign: '=';
QuestionMark: '?';
Colon: ':';
Ellipsis: '...';
Dot: '.';
PlusPlus: '++';
MinusMinus: '--';
Plus: '+';
Minus: '-';
BitNot: '~';
Not: '!';
Multiply: '*';
Divide: '/';
Modulus: '%';
RightShiftArithmetic: '>>';
LeftShiftArithmetic: '<<';
RightShiftLogical: '>>>';
LessThan: '<';
MoreThan: '>';
LessThanEquals: '<=';
GreaterThanEquals: '>=';
Equals_: '==';
NotEquals: '!=';
IdentityEquals: '===';
IdentityNotEquals: '!==';
BitAnd: '&';
BitXOr: '^';
BitOr: '|';
And: '&&';
Or: '||';
MultiplyAssign: '*=';
DivideAssign: '/=';
ModulusAssign: '%=';
PlusAssign: '+=';
MinusAssign: '-=';
LeftShiftArithmeticAssign: '<<=';
RightShiftArithmeticAssign: '>>=';
RightShiftLogicalAssign: '>>>=';
BitAndAssign: '&=';
BitXorAssign: '^=';
BitOrAssign: '|=';
ARROW: '=>';
/// Null Literals
NullLiteral: 'null';
/// Boolean Literals
BooleanLiteral: 'true'
| 'false';
/// Numeric Literals
DecimalLiteral: DecimalIntegerLiteral '.' [0-9]* ExponentPart?
| '.' [0-9]+ ExponentPart?
| DecimalIntegerLiteral ExponentPart?
;
/// Numeric Literals
HexIntegerLiteral: '0' [xX] HexDigit+;
OctalIntegerLiteral: '0' [0-7]+ {!this.IsStrictMode()}?;
OctalIntegerLiteral2: '0' [oO] [0-7]+;
BinaryIntegerLiteral: '0' [bB] [01]+;
/// Keywords
Break: 'break';
Do: 'do';
Instanceof: 'instanceof';
Typeof: 'typeof';
Case: 'case';
Else: 'else';
New: 'new';
Var: 'var';
Catch: 'catch';
Finally: 'finally';
Return: 'return';
Void: 'void';
Continue: 'continue';
For: 'for';
Switch: 'switch';
While: 'while';
Debugger: 'debugger';
Function_: 'function';
This: 'this';
With: 'with';
Default: 'default';
If: 'if';
Throw: 'throw';
Delete: 'delete';
In: 'in';
Try: 'try';
As: 'as';
From: 'from';
ReadOnly: 'readonly';
Async: 'async';
/// Future Reserved Words
Class: 'class';
Enum: 'enum';
Extends: 'extends';
Super: 'super';
Const: 'const';
Export: 'export';
Import: 'import';
/// The following tokens are also considered to be FutureReservedWords
/// when parsing strict mode
Implements: 'implements' ;
Let: 'let' ;
Private: 'private' ;
Public: 'public' ;
Interface: 'interface' ;
Package: 'package' ;
Protected: 'protected' ;
Static: 'static' ;
Yield: 'yield' ;
//keywords:
Any : 'any';
Number: 'number';
Boolean: 'boolean';
String: 'string';
Symbol: 'symbol';
TypeAlias : 'type';
Get: 'get';
Set: 'set';
Constructor: 'constructor';
Namespace: 'namespace';
Require: 'require';
Module: 'module';
Declare: 'declare';
Abstract: 'abstract';
Is: 'is';
//
// Ext.2 Additions to 1.8: Decorators
//
At: '@';
/// Identifier Names and Identifiers
Identifier: IdentifierStart IdentifierPart*;
/// String Literals
StringLiteral: ('"' DoubleStringCharacter* '"'
| '\'' SingleStringCharacter* '\'') {this.ProcessStringLiteral();}
;
BackTick: '`' {this.IncreaseTemplateDepth();} -> pushMode(TEMPLATE);
WhiteSpaces: [\t\u000B\u000C\u0020\u00A0]+ -> channel(HIDDEN);
LineTerminator: [\r\n\u2028\u2029] -> channel(HIDDEN);
/// Comments
HtmlComment: '<!--' .*? '-->' -> channel(HIDDEN);
CDataComment: '<![CDATA[' .*? ']]>' -> channel(HIDDEN);
UnexpectedCharacter: . -> channel(ERROR);
mode TEMPLATE;
TemplateStringEscapeAtom: '\\' .;
BackTickInside: '`' {this.DecreaseTemplateDepth();} -> type(BackTick), popMode;
TemplateStringStartExpression: '${' {this.StartTemplateString();} -> pushMode(DEFAULT_MODE);
TemplateStringAtom: ~[`\\];
// Fragment rules
fragment DoubleStringCharacter
: ~["\\\r\n]
| '\\' EscapeSequence
| LineContinuation
;
fragment SingleStringCharacter
: ~['\\\r\n]
| '\\' EscapeSequence
| LineContinuation
;
fragment EscapeSequence
: CharacterEscapeSequence
| '0' // no digit ahead! TODO
| HexEscapeSequence
| UnicodeEscapeSequence
| ExtendedUnicodeEscapeSequence
;
fragment CharacterEscapeSequence
: SingleEscapeCharacter
| NonEscapeCharacter
;
fragment HexEscapeSequence
: 'x' HexDigit HexDigit
;
fragment UnicodeEscapeSequence
: 'u' HexDigit HexDigit HexDigit HexDigit
;
fragment ExtendedUnicodeEscapeSequence
: 'u' '{' HexDigit+ '}'
;
fragment SingleEscapeCharacter
: ['"\\bfnrtv]
;
fragment NonEscapeCharacter
: ~['"\\bfnrtv0-9xu\r\n]
;
fragment EscapeCharacter
: SingleEscapeCharacter
| [0-9]
| [xu]
;
fragment LineContinuation
: '\\' [\r\n\u2028\u2029]
;
fragment HexDigit
: [0-9a-fA-F]
;
fragment DecimalIntegerLiteral
: '0'
| [1-9] [0-9]*
;
fragment ExponentPart
: [eE] [+-]? [0-9]+
;
fragment IdentifierPart
: IdentifierStart
| [\p{Mn}]
| [\p{Nd}]
| [\p{Pc}]
| '\u200C'
| '\u200D'
;
fragment IdentifierStart
: [\p{L}]
| [$_]
| '\\' UnicodeEscapeSequence
;
fragment RegularExpressionFirstChar
: ~[*\r\n\u2028\u2029\\/[]
| RegularExpressionBackslashSequence
| '[' RegularExpressionClassChar* ']'
;
fragment RegularExpressionChar
: ~[\r\n\u2028\u2029\\/[]
| RegularExpressionBackslashSequence
| '[' RegularExpressionClassChar* ']'
;
fragment RegularExpressionClassChar
: ~[\r\n\u2028\u2029\]\\]
| RegularExpressionBackslashSequence
;
fragment RegularExpressionBackslashSequence
: '\\' ~[\r\n\u2028\u2029]
;

854
codegpt-core/src/main/antlr/TypeScriptParser.g4
View file

@ -1,854 +0,0 @@
/*
* The MIT License (MIT)
*
* Copyright (c) 2014 by Bart Kiers (original author) and Alexandre Vitorelli (contributor -> ported to CSharp)
* Copyright (c) 2017 by Ivan Kochurkin (Positive Technologies):
added ECMAScript 6 support, cleared and transformed to the universal grammar.
* Copyright (c) 2018 by Juan Alvarez (contributor -> ported to Go)
* Copyright (c) 2019 by Andrii Artiushok (contributor -> added TypeScript support)
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
parser grammar TypeScriptParser;
options {
tokenVocab=TypeScriptLexer;
superClass=TypeScriptParserBase;
}
// SupportSyntax
initializer
: '=' singleExpression
;
bindingPattern
: (arrayLiteral | objectLiteral)
;
// TypeScript SPart
// A.1 Types
typeParameters
: '<' typeParameterList? '>'
;
typeParameterList
: typeParameter (',' typeParameter)*
;
typeParameter
: Identifier constraint?
| typeParameters
;
constraint
: 'extends' type_
;
typeArguments
: '<' typeArgumentList? '>'
;
typeArgumentList
: typeArgument (',' typeArgument)*
;
typeArgument
: type_
;
type_
: unionOrIntersectionOrPrimaryType
| functionType
| constructorType
| typeGeneric
| StringLiteral
;
unionOrIntersectionOrPrimaryType
: unionOrIntersectionOrPrimaryType '|' unionOrIntersectionOrPrimaryType #Union
| unionOrIntersectionOrPrimaryType '&' unionOrIntersectionOrPrimaryType #Intersection
| primaryType #Primary
;
primaryType
: '(' type_ ')' #ParenthesizedPrimType
| predefinedType #PredefinedPrimType
| typeReference #ReferencePrimType
| objectType #ObjectPrimType
| primaryType {notLineTerminator()}? '[' ']' #ArrayPrimType
| '[' tupleElementTypes ']' #TuplePrimType
| typeQuery #QueryPrimType
| This #ThisPrimType
| typeReference Is primaryType #RedefinitionOfType
;
predefinedType
: Any
| Number
| Boolean
| String
| Symbol
| Void
;
typeReference
: typeName nestedTypeGeneric?
;
nestedTypeGeneric
: typeIncludeGeneric
| typeGeneric
;
// I tried recursive include, but it's not working.
// typeGeneric
// : '<' typeArgumentList typeGeneric?'>'
// ;
//
// TODO: Fix recursive
//
typeGeneric
: '<' typeArgumentList '>'
;
typeIncludeGeneric
:'<' typeArgumentList '<' typeArgumentList ('>' bindingPattern '>' | '>>')
;
typeName
: Identifier
| namespaceName
;
objectType
: '{' typeBody? '}'
;
typeBody
: typeMemberList (SemiColon | ',')?
;
typeMemberList
: typeMember ((SemiColon | ',') typeMember)*
;
typeMember
: propertySignatur
| callSignature
| constructSignature
| indexSignature
| methodSignature ('=>' type_)?
;
arrayType
: primaryType {notLineTerminator()}? '[' ']'
;
tupleType
: '[' tupleElementTypes ']'
;
tupleElementTypes
: type_ (',' type_)*
;
functionType
: typeParameters? '(' parameterList? ')' '=>' type_
;
constructorType
: 'new' typeParameters? '(' parameterList? ')' '=>' type_
;
typeQuery
: 'typeof' typeQueryExpression
;
typeQueryExpression
: Identifier
| (identifierName '.')+ identifierName
;
propertySignatur
: ReadOnly? propertyName '?'? typeAnnotation? ('=>' type_)?
;
typeAnnotation
: ':' type_
;
callSignature
: typeParameters? '(' parameterList? ')' typeAnnotation?
;
parameterList
: restParameter
| parameter (',' parameter)* (',' restParameter)?
;
requiredParameterList
: requiredParameter (',' requiredParameter)*
;
parameter
: requiredParameter
| optionalParameter
;
optionalParameter
: decoratorList? ( accessibilityModifier? identifierOrPattern ('?' typeAnnotation? | typeAnnotation? initializer))
;
restParameter
: '...' singleExpression typeAnnotation?
;
requiredParameter
: decoratorList? accessibilityModifier? identifierOrPattern typeAnnotation?
;
accessibilityModifier
: Public
| Private
| Protected
;
identifierOrPattern
: identifierName
| bindingPattern
;
constructSignature
: 'new' typeParameters? '(' parameterList? ')' typeAnnotation?
;
indexSignature
: '[' Identifier ':' (Number|String) ']' typeAnnotation
;
methodSignature
: propertyName '?'? callSignature
;
typeAliasDeclaration
: 'type' Identifier typeParameters? '=' type_ SemiColon
;
constructorDeclaration
: accessibilityModifier? Constructor '(' formalParameterList? ')' ( ('{' functionBody '}') | SemiColon)?
;
// A.5 Interface
interfaceDeclaration
: Export? Declare? Interface Identifier typeParameters? interfaceExtendsClause? objectType SemiColon?
;
interfaceExtendsClause
: Extends classOrInterfaceTypeList
;
classOrInterfaceTypeList
: typeReference (',' typeReference)*
;
// A.7 Interface
enumDeclaration
: Const? Enum Identifier '{' enumBody? '}'
;
enumBody
: enumMemberList ','?
;
enumMemberList
: enumMember (',' enumMember)*
;
enumMember
: propertyName ('=' singleExpression)?
;
// A.8 Namespaces
namespaceDeclaration
: Namespace namespaceName '{' statementList? '}'
;
namespaceName
: Identifier ('.'+ Identifier)*
;
importAliasDeclaration
: Identifier '=' namespaceName SemiColon
;
// Ext.2 Additions to 1.8: Decorators
decoratorList
: decorator+ ;
decorator
: '@' (decoratorMemberExpression | decoratorCallExpression)
;
decoratorMemberExpression
: Identifier
| decoratorMemberExpression '.' identifierName
| '(' singleExpression ')'
;
decoratorCallExpression
: decoratorMemberExpression arguments;
// ECMAPart
program
: sourceElements? EOF
;
sourceElement
: Export? statement
;
statement
: block
| importStatement
| exportStatement
| emptyStatement_
| abstractDeclaration //ADDED
| classDeclaration
| interfaceDeclaration //ADDED
| namespaceDeclaration //ADDED
| ifStatement
| iterationStatement
| continueStatement
| breakStatement
| returnStatement
| yieldStatement
| withStatement
| labelledStatement
| switchStatement
| throwStatement
| tryStatement
| debuggerStatement
| functionDeclaration
| arrowFunctionDeclaration
| generatorFunctionDeclaration
| variableStatement
| typeAliasDeclaration //ADDED
| enumDeclaration //ADDED
| expressionStatement
| Export statement
;
block
: '{' statementList? '}'
;
statementList
: statement+
;
abstractDeclaration
: Abstract (Identifier callSignature | variableStatement) eos
;
importStatement
: Import (fromBlock | importAliasDeclaration)
;
fromBlock
: (Multiply | multipleImportStatement) (As identifierName)? From StringLiteral eos
;
multipleImportStatement
: (identifierName ',')? '{' identifierName (',' identifierName)* '}'
;
exportStatement
: Export Default? (fromBlock | statement)
;
variableStatement
: bindingPattern typeAnnotation? initializer SemiColon?
| accessibilityModifier? varModifier? ReadOnly? variableDeclarationList SemiColon?
| Declare varModifier? variableDeclarationList SemiColon?
;
variableDeclarationList
: variableDeclaration (',' variableDeclaration)*
;
variableDeclaration
: ( identifierOrKeyWord | arrayLiteral | objectLiteral) typeAnnotation? singleExpression? ('=' typeParameters? singleExpression)? // ECMAScript 6: Array & Object Matching
;
emptyStatement_
: SemiColon
;
expressionStatement
: {this.notOpenBraceAndNotFunction()}? expressionSequence SemiColon?
;
ifStatement
: If '(' expressionSequence ')' statement (Else statement)?
;
iterationStatement
: Do statement While '(' expressionSequence ')' eos # DoStatement
| While '(' expressionSequence ')' statement # WhileStatement
| For '(' expressionSequence? SemiColon expressionSequence? SemiColon expressionSequence? ')' statement # ForStatement
| For '(' varModifier variableDeclarationList SemiColon expressionSequence? SemiColon expressionSequence? ')'
statement # ForVarStatement
| For '(' singleExpression (In | Identifier{this.p("of")}?) expressionSequence ')' statement # ForInStatement
| For '(' varModifier variableDeclaration (In | Identifier{this.p("of")}?) expressionSequence ')' statement # ForVarInStatement
;
varModifier
: Var
| Let
| Const
;
continueStatement
: Continue ({this.notLineTerminator()}? Identifier)? eos
;
breakStatement
: Break ({this.notLineTerminator()}? Identifier)? eos
;
returnStatement
: Return ({this.notLineTerminator()}? expressionSequence)? eos
;
yieldStatement
: Yield ({this.notLineTerminator()}? expressionSequence)? eos
;
withStatement
: With '(' expressionSequence ')' statement
;
switchStatement
: Switch '(' expressionSequence ')' caseBlock
;
caseBlock
: '{' caseClauses? (defaultClause caseClauses?)? '}'
;
caseClauses
: caseClause+
;
caseClause
: Case expressionSequence ':' statementList?
;
defaultClause
: Default ':' statementList?
;
labelledStatement
: Identifier ':' statement
;
throwStatement
: Throw {this.notLineTerminator()}? expressionSequence eos
;
tryStatement
: Try block (catchProduction finallyProduction? | finallyProduction)
;
catchProduction
: Catch '(' Identifier ')' block
;
finallyProduction
: Finally block
;
debuggerStatement
: Debugger eos
;
functionDeclaration
: Function_ Identifier callSignature ( ('{' functionBody '}') | SemiColon)
;
//Ovveride ECMA
classDeclaration
: decoratorList? (Export Default?)? Abstract? Class Identifier typeParameters? classHeritage classTail
;
classHeritage
: classExtendsClause? implementsClause?
;
classTail
: '{' classElement* '}'
;
classExtendsClause
: Extends typeReference
;
implementsClause
: Implements classOrInterfaceTypeList
;
// Classes modified
classElement
: constructorDeclaration
| decoratorList? propertyMemberDeclaration
| indexMemberDeclaration
| statement
;
propertyMemberDeclaration
: propertyMemberBase propertyName '?'? typeAnnotation? initializer? SemiColon # PropertyDeclarationExpression
| propertyMemberBase propertyName callSignature ( ('{' functionBody '}') | SemiColon) # MethodDeclarationExpression
| propertyMemberBase (getAccessor | setAccessor) # GetterSetterDeclarationExpression
| abstractDeclaration # AbstractMemberDeclaration
;
propertyMemberBase
: accessibilityModifier? Async? Static? ReadOnly?
;
indexMemberDeclaration
: indexSignature SemiColon
;
generatorMethod
: '*'? Identifier '(' formalParameterList? ')' '{' functionBody '}'
;
generatorFunctionDeclaration
: Function_ '*' Identifier? '(' formalParameterList? ')' '{' functionBody '}'
;
generatorBlock
: '{' generatorDefinition (',' generatorDefinition)* ','? '}'
;
generatorDefinition
: '*' iteratorDefinition
;
iteratorBlock
: '{' iteratorDefinition (',' iteratorDefinition)* ','? '}'
;
iteratorDefinition
: '[' singleExpression ']' '(' formalParameterList? ')' '{' functionBody '}'
;
formalParameterList
: formalParameterArg (',' formalParameterArg)* (',' lastFormalParameterArg)?
| lastFormalParameterArg
| arrayLiteral // ECMAScript 6: Parameter Context Matching
| objectLiteral (':' formalParameterList)? // ECMAScript 6: Parameter Context Matching
;
formalParameterArg
: decorator? accessibilityModifier? identifierOrKeyWord '?'? typeAnnotation? ('=' singleExpression)? // ECMAScript 6: Initialization
;
lastFormalParameterArg // ECMAScript 6: Rest Parameter
: Ellipsis Identifier typeAnnotation?
;
functionBody
: sourceElements?
;
sourceElements
: sourceElement+
;
arrayLiteral
: ('[' elementList? ']')
;
elementList
: arrayElement (','+ arrayElement)*
;
arrayElement // ECMAScript 6: Spread Operator
: Ellipsis? (singleExpression | Identifier) ','?
;
objectLiteral
: '{' (propertyAssignment (',' propertyAssignment)* ','?)? '}'
;
// MODIFIED
propertyAssignment
: propertyName (':' |'=') singleExpression # PropertyExpressionAssignment
| '[' singleExpression ']' ':' singleExpression # ComputedPropertyExpressionAssignment
| getAccessor # PropertyGetter
| setAccessor # PropertySetter
| generatorMethod # MethodProperty
| identifierOrKeyWord # PropertyShorthand
| restParameter # RestParameterInObject
;
getAccessor
: getter '(' ')' typeAnnotation? '{' functionBody '}'
;
setAccessor
: setter '(' ( Identifier | bindingPattern) typeAnnotation? ')' '{' functionBody '}'
;
propertyName
: identifierName
| StringLiteral
| numericLiteral
;
arguments
: '(' (argumentList ','?)? ')'
;
argumentList
: argument (',' argument)*
;
argument // ECMAScript 6: Spread Operator
: Ellipsis? (singleExpression | Identifier)
;
expressionSequence
: singleExpression (',' singleExpression)*
;
functionExpressionDeclaration
: Function_ Identifier? '(' formalParameterList? ')' typeAnnotation? '{' functionBody '}'
;
singleExpression
: functionExpressionDeclaration # FunctionExpression
| arrowFunctionDeclaration # ArrowFunctionExpression // ECMAScript 6
| singleExpression '[' expressionSequence ']' # MemberIndexExpression
| singleExpression '!'? '.' identifierName nestedTypeGeneric? # MemberDotExpression
// Split to try `new Date()` first, then `new Date`.
| New singleExpression typeArguments? arguments # NewExpression
| New singleExpression typeArguments? # NewExpression
| singleExpression arguments # ArgumentsExpression
| singleExpression {this.notLineTerminator()}? '++' # PostIncrementExpression
| singleExpression {this.notLineTerminator()}? '--' # PostDecreaseExpression
| Delete singleExpression # DeleteExpression
| Void singleExpression # VoidExpression
| Typeof singleExpression # TypeofExpression
| '++' singleExpression # PreIncrementExpression
| '--' singleExpression # PreDecreaseExpression
| '+' singleExpression # UnaryPlusExpression
| '-' singleExpression # UnaryMinusExpression
| '~' singleExpression # BitNotExpression
| '!' singleExpression # NotExpression
| singleExpression ('*' | '/' | '%') singleExpression # MultiplicativeExpression
| singleExpression ('+' | '-') singleExpression # AdditiveExpression
| singleExpression ('<<' | '>>' | '>>>') singleExpression # BitShiftExpression
| singleExpression ('<' | '>' | '<=' | '>=') singleExpression # RelationalExpression
| singleExpression Instanceof singleExpression # InstanceofExpression
| singleExpression In singleExpression # InExpression
| singleExpression ('==' | '!=' | '===' | '!==') singleExpression # EqualityExpression
| singleExpression '&' singleExpression # BitAndExpression
| singleExpression '^' singleExpression # BitXOrExpression
| singleExpression '|' singleExpression # BitOrExpression
| singleExpression '&&' singleExpression # LogicalAndExpression
| singleExpression '||' singleExpression # LogicalOrExpression
| singleExpression '?' singleExpression ':' singleExpression # TernaryExpression
| singleExpression '=' singleExpression # AssignmentExpression
| singleExpression assignmentOperator singleExpression # AssignmentOperatorExpression
| singleExpression templateStringLiteral # TemplateStringExpression // ECMAScript 6
| iteratorBlock # IteratorsExpression // ECMAScript 6
| generatorBlock # GeneratorsExpression // ECMAScript 6
| generatorFunctionDeclaration # GeneratorsFunctionExpression // ECMAScript 6
| yieldStatement # YieldExpression // ECMAScript 6
| This # ThisExpression
| identifierName singleExpression? # IdentifierExpression
| Super # SuperExpression
| literal # LiteralExpression
| arrayLiteral # ArrayLiteralExpression
| objectLiteral # ObjectLiteralExpression
| '(' expressionSequence ')' # ParenthesizedExpression
| typeArguments expressionSequence? # GenericTypes
| singleExpression As asExpression # CastAsExpression
;
asExpression
: predefinedType ('[' ']')?
| singleExpression
;
arrowFunctionDeclaration
: Async? arrowFunctionParameters typeAnnotation? '=>' arrowFunctionBody
;
arrowFunctionParameters
: Identifier
| '(' formalParameterList? ')'
;
arrowFunctionBody
: singleExpression
| '{' functionBody '}'
;
assignmentOperator
: '*='
| '/='
| '%='
| '+='
| '-='
| '<<='
| '>>='
| '>>>='
| '&='
| '^='
| '|='
;
literal
: NullLiteral
| BooleanLiteral
| StringLiteral
| templateStringLiteral
| RegularExpressionLiteral
| numericLiteral
;
templateStringLiteral
: BackTick templateStringAtom* BackTick
;
templateStringAtom
: TemplateStringAtom
| TemplateStringStartExpression singleExpression TemplateCloseBrace
| TemplateStringEscapeAtom
;
numericLiteral
: DecimalLiteral
| HexIntegerLiteral
| OctalIntegerLiteral
| OctalIntegerLiteral2
| BinaryIntegerLiteral
;
identifierName
: Identifier
| reservedWord
;
identifierOrKeyWord
: Identifier
| TypeAlias
| Require
;
reservedWord
: keyword
| NullLiteral
| BooleanLiteral
;
keyword
: Break
| Do
| Instanceof
| Typeof
| Case
| Else
| New
| Var
| Catch
| Finally
| Return
| Void
| Continue
| For
| Switch
| While
| Debugger
| Function_
| This
| With
| Default
| If
| Throw
| Delete
| In
| Try
| ReadOnly
| Async
| From
| Class
| Enum
| Extends
| Super
| Const
| Export
| Import
| Implements
| Let
| Private
| Public
| Interface
| Package
| Protected
| Static
| Yield
| Get
| Set
| Require
| TypeAlias
| String
| Boolean
| Number
| Module
;
getter
: Get propertyName
;
setter
: Set propertyName
;
eos
: SemiColon
| EOF
| {this.lineTerminatorAhead()}?
| {this.closeBrace()}?
;

1647
codegpt-core/src/main/antlr/UnicodeClasses.g4
View file

File diff suppressed because it is too large Load diff

146
codegpt-core/src/main/java/ee/carlrobert/embedding/EmbeddingsService.java
View file

@ -1,146 +0,0 @@
package ee.carlrobert.embedding;
import static com.github.jelmerk.knn.util.VectorUtils.normalize;
import static java.util.stream.Collectors.toList;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.github.jelmerk.knn.Item;
import com.github.jelmerk.knn.SearchResult;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import ee.carlrobert.llm.client.openai.OpenAIClient;
import ee.carlrobert.llm.client.openai.completion.OpenAIChatCompletionModel;
import ee.carlrobert.llm.client.openai.completion.request.OpenAIChatCompletionMessage;
import ee.carlrobert.llm.client.openai.completion.request.OpenAIChatCompletionRequest;
import ee.carlrobert.splitter.SplitterFactory;
import ee.carlrobert.vector.VectorStore;
import ee.carlrobert.vector.Word;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.jetbrains.annotations.Nullable;
public class EmbeddingsService {
private static final Logger LOG = Logger.getInstance(EmbeddingsService.class);
private final VectorStore vectorStore;
private final OpenAIClient openAIClient;
public EmbeddingsService(OpenAIClient openAIClient, Path pluginBasePath) {
this.openAIClient = openAIClient;
this.vectorStore = VectorStore.getInstance(pluginBasePath);
}
public List<double[]> getEmbeddings(List<String> chunks) {
return openAIClient.getEmbeddings(chunks);
}
public String buildPromptWithContext(String prompt) {
try {
var inputEmbedding = openAIClient.getEmbedding(getSearchQuery(prompt));
var sortedResult = vectorStore.loadIndex()
.findNearest(normalize(inputEmbedding), 10)
.stream()
.map(SearchResult::item)
.sorted(Comparator.comparing(Word::getMeta))
.collect(toList());
var context = sortedResult.stream().map(Word::id).collect(Collectors.joining());
var fileNames = sortedResult.stream().map(Word::getMeta).collect(Collectors.toSet());
return getResourceContent("/prompts/prompt-with-context.txt")
.replace("{prompt}", prompt)
.replace("{context}", new GeneratedContextDetails(context, fileNames).getContext());
} catch (IOException e) {
LOG.error("Unable to load vector index", e);
return prompt;
}
}
public List<Item<Object, double[]>> createEmbeddings(
List<ReferencedFile> referencedFiles,
@Nullable ProgressIndicator indicator) {
var words = new ArrayList<Item<Object, double[]>>();
for (int i = 0; i < referencedFiles.size(); i++) {
try {
var referencedFile = referencedFiles.get(i);
addEmbeddings(referencedFile, words);
if (indicator != null) {
indicator.setFraction((double) i / referencedFiles.size());
}
} catch (Throwable t) {
// ignore
}
}
return words;
}
private String getSearchQuery(String userPrompt) throws JsonProcessingException {
var message = new OpenAIChatCompletionMessage(
"user",
getResourceContent("/prompts/text-generator.txt")
.replace("{prompt}", userPrompt));
var request = new OpenAIChatCompletionRequest.Builder(List.of(message))
.setModel(OpenAIChatCompletionModel.GPT_4)
.setMaxTokens(400)
.setTemperature(0.1)
.setStream(false)
.build();
return openAIClient.getChatCompletion(request)
.getChoices()
.get(0)
.getMessage()
.getContent();
}
private void addEmbeddings(
ReferencedFile referencedFile,
List<Item<Object, double[]>> prevEmbeddings) {
var fileExtension = referencedFile.getFileExtension();
var codeSplitter = SplitterFactory.getCodeSplitter(fileExtension);
if (codeSplitter != null) {
var chunks = codeSplitter.split(
referencedFile.getFileName(),
referencedFile.getFileContent());
var embeddings = openAIClient.getEmbeddings(chunks);
for (int i = 0; i < chunks.size(); i++) {
prevEmbeddings.add(
new Word(chunks.get(i), referencedFile.getFileName(), normalize(embeddings.get(i))));
}
} else {
var chunks = splitText(referencedFile.getFileContent(), 400);
var embeddings = getEmbeddings(chunks);
for (int i = 0; i < chunks.size(); i++) {
prevEmbeddings.add(
new Word(chunks.get(i), referencedFile.getFileName(), normalize(embeddings.get(i))));
}
}
}
private static List<String> splitText(String str, int chunkSize) {
int len = str.length();
var chunks = new ArrayList<String>();
for (int i = 0; i < len; i += chunkSize) {
chunks.add(str.substring(i, Math.min(len, i + chunkSize)));
}
return chunks;
}
// TODO: Move to shared module
private static String getResourceContent(String name) {
try (var stream =
Objects.requireNonNull(EmbeddingsService.class.getResourceAsStream(name))) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeException("Unable to read resource", e);
}
}
}

22
codegpt-core/src/main/java/ee/carlrobert/embedding/GeneratedContextDetails.java
View file

@ -1,22 +0,0 @@
package ee.carlrobert.embedding;
import java.util.Set;
public class GeneratedContextDetails {
private final String context;
private final Set<String> fileNames;
public GeneratedContextDetails(String context, Set<String> fileNames) {
this.context = context;
this.fileNames = fileNames;
}
public String getContext() {
return context;
}
public Set<String> getFileNames() {
return fileNames;
}
}

34
codegpt-core/src/main/java/ee/carlrobert/splitter/CodeSplitter.java
View file

@ -1,34 +0,0 @@
package ee.carlrobert.splitter;
import java.util.ArrayList;
import java.util.List;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
abstract class CodeSplitter implements Splitter {
protected List<String> chunks = new ArrayList<>();
protected abstract ParseTree getParseTree(CodePointCharStream charStream);
protected abstract ParseTreeListener getParseTreeListener();
protected String parseContext(ParserRuleContext ctx) {
return ctx.start.getInputStream().getText(
new Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex()));
}
@Override
public List<String> split(String fileName, String content) {
chunks = new ArrayList<>();
ParseTreeWalker.DEFAULT.walk(
getParseTreeListener(),
getParseTree(CharStreams.fromString(content)));
return chunks;
}
}

32
codegpt-core/src/main/java/ee/carlrobert/splitter/JavaCodeSplitter.java
View file

@ -1,32 +0,0 @@
package ee.carlrobert.splitter;
import grammar.JavaLexer;
import grammar.JavaParser;
import grammar.JavaParserBaseListener;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
public class JavaCodeSplitter extends CodeSplitter {
@Override
protected ParseTree getParseTree(CodePointCharStream charStream) {
return new JavaParser(new CommonTokenStream(new JavaLexer(charStream))).compilationUnit();
}
@Override
protected ParseTreeListener getParseTreeListener() {
return new JavaParserBaseListener() {
@Override
public void enterConstructorDeclaration(JavaParser.ConstructorDeclarationContext ctx) {
chunks.add(parseContext(ctx));
}
@Override
public void enterMethodDeclaration(JavaParser.MethodDeclarationContext ctx) {
chunks.add(parseContext(ctx));
}
};
}
}

28
codegpt-core/src/main/java/ee/carlrobert/splitter/JsonSplitter.java
View file

@ -1,28 +0,0 @@
package ee.carlrobert.splitter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.intellij.openapi.diagnostic.Logger;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONObject;
public class JsonSplitter implements Splitter {
private static final Logger LOG = Logger.getInstance(JsonSplitter.class);
@Override
public List<String> split(String fileName, String content) {
var chunks = new ArrayList<String>();
try {
// TODO: Switch to ObjectMapper
for (var entry : new JSONObject(content).toMap().entrySet()) {
chunks.add(new ObjectMapper().writeValueAsString(entry));
}
} catch (JsonProcessingException e) {
LOG.error("Something went wrong while chunking the json", e);
}
return chunks;
}
}

27
codegpt-core/src/main/java/ee/carlrobert/splitter/PythonCodeSplitter.java
View file

@ -1,27 +0,0 @@
package ee.carlrobert.splitter;
import grammar.PythonLexer;
import grammar.PythonParser;
import grammar.PythonParserBaseListener;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
public class PythonCodeSplitter extends CodeSplitter {
@Override
protected ParseTree getParseTree(CodePointCharStream charStream) {
return new PythonParser(new CommonTokenStream(new PythonLexer(charStream))).file_input();
}
@Override
protected ParseTreeListener getParseTreeListener() {
return new PythonParserBaseListener() {
@Override
public void enterClass_or_func_def_stmt(PythonParser.Class_or_func_def_stmtContext ctx) {
chunks.add(parseContext(ctx));
}
};
}
}

8
codegpt-core/src/main/java/ee/carlrobert/splitter/Splitter.java
View file

@ -1,8 +0,0 @@
package ee.carlrobert.splitter;
import java.util.List;
public interface Splitter {
List<String> split(String fileName, String content);
}

22
codegpt-core/src/main/java/ee/carlrobert/splitter/SplitterFactory.java
View file

@ -1,22 +0,0 @@
package ee.carlrobert.splitter;
import org.jetbrains.annotations.Nullable;
public class SplitterFactory {
public static @Nullable Splitter getCodeSplitter(String fileExtension) {
switch (fileExtension) {
case "java":
return new JavaCodeSplitter();
case "py":
return new PythonCodeSplitter();
case "json":
return new JsonSplitter();
case "ts":
case "tsx":
return new TypeScriptCodeSplitter();
default:
return null;
}
}
}

22
codegpt-core/src/main/java/ee/carlrobert/splitter/TypeScriptCodeSplitter.java
View file

@ -1,22 +0,0 @@
package ee.carlrobert.splitter;
import grammar.TypeScriptLexer;
import grammar.TypeScriptParser;
import grammar.TypeScriptParserBaseListener;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
public class TypeScriptCodeSplitter extends CodeSplitter {
@Override
protected ParseTree getParseTree(CodePointCharStream charStream) {
return new TypeScriptParser(new CommonTokenStream(new TypeScriptLexer(charStream))).program();
}
@Override
protected ParseTreeListener getParseTreeListener() {
return new TypeScriptParserBaseListener() {};
}
}

63
codegpt-core/src/main/java/ee/carlrobert/vector/VectorStore.java
View file

@ -1,63 +0,0 @@
package ee.carlrobert.vector;
import com.github.jelmerk.knn.DistanceFunctions;
import com.github.jelmerk.knn.Item;
import com.github.jelmerk.knn.hnsw.HnswIndex;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.util.io.FileUtil;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
public class VectorStore {
private static VectorStore instance;
private final String storePath;
private VectorStore(Path pluginPath) {
this.storePath = getIndexStorePath(pluginPath.toString());
}
public static VectorStore getInstance(Path pluginPath) {
if (instance == null) {
instance = new VectorStore(pluginPath);
}
return instance;
}
public HnswIndex<Object, double[], Word, Object> loadIndex() throws IOException {
return loadIndex(storePath);
}
public HnswIndex<Object, double[], Word, Object> loadIndex(String path) throws IOException {
return HnswIndex.load(new File(path), this.getClass().getClassLoader());
}
public void save(List<Item<Object, double[]>> words) {
var hnswIndex = HnswIndex
.newBuilder(
words.get(0).vector().length,
DistanceFunctions.DOUBLE_COSINE_DISTANCE,
words.size())
.build();
try {
hnswIndex.addAll(words);
hnswIndex.save(new File(storePath));
} catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
}
}
public boolean isIndexExists() {
return FileUtil.exists(storePath);
}
private String getIndexStorePath(String pluginBasePath) {
if (ApplicationManager.getApplication().isUnitTestMode()) {
pluginBasePath = new File("src/test/resources/indexes").getAbsolutePath();
}
return pluginBasePath + File.separator + "hnsw.index";
}
}

46
codegpt-core/src/main/java/ee/carlrobert/vector/Word.java
View file

@ -1,46 +0,0 @@
package ee.carlrobert.vector;
import com.github.jelmerk.knn.Item;
import java.util.Arrays;
public class Word implements Item<Object, double[]> {
private static final long serialVersionUID = 1L;
private final String id;
private final String meta;
private final double[] vector;
public Word(String id, String meta, double[] vector) {
this.id = id;
this.meta = meta;
this.vector = vector;
}
@Override
public String id() {
return id;
}
@Override
public double[] vector() {
return vector;
}
@Override
public int dimensions() {
return vector.length;
}
@Override
public String toString() {
return "Word{"
+ "id='" + id + '\''
+ ", vector=" + Arrays.toString(vector)
+ '}';
}
public String getMeta() {
return meta;
}
}

184
codegpt-core/src/main/java/grammar/PythonLexerBase.java
View file

@ -1,184 +0,0 @@
package grammar;
import java.util.ArrayDeque;
import java.util.Deque;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
public abstract class PythonLexerBase extends Lexer {
public static int TabSize = 8;
// The amount of opened braces, brackets and parenthesis.
private int _opened;
// The stack that keeps track of the indentation level.
private final Deque<Integer> _indents = new ArrayDeque<>();
// A circular buffer where extra tokens are pushed on (see the NEWLINE and WS lexer rules).
private int _firstTokensInd;
private int _lastTokenInd;
private Token[] _buffer = new Token[32];
private Token _lastToken;
protected PythonLexerBase(CharStream input) {
super(input);
}
@Override
public void emit(Token token) {
super.setToken(token);
if (_buffer[_firstTokensInd] != null)
{
_lastTokenInd = IncTokenInd(_lastTokenInd);
if (_lastTokenInd == _firstTokensInd)
{
// Enlarge buffer
Token[] newArray = new Token[_buffer.length * 2];
int destInd = newArray.length - (_buffer.length - _firstTokensInd);
System.arraycopy(_buffer, 0, newArray, 0, _firstTokensInd);
System.arraycopy(_buffer, _firstTokensInd, newArray, destInd, _buffer.length - _firstTokensInd);
_firstTokensInd = destInd;
_buffer = newArray;
}
}
_buffer[_lastTokenInd] = token;
_lastToken = token;
}
@Override
public Token nextToken() {
// Check if the end-of-file is ahead and there are still some DEDENTS expected.
if (_input.LA(1) == EOF && _indents.size() > 0)
{
if (_buffer[_lastTokenInd] == null || _buffer[_lastTokenInd].getType() != PythonLexer.LINE_BREAK)
{
// First emit an extra line break that serves as the end of the statement.
emit(PythonLexer.LINE_BREAK);
}
// Now emit as much DEDENT tokens as needed.
while (_indents.size() != 0)
{
emit(PythonLexer.DEDENT);
_indents.pop();
}
}
Token next = super.nextToken();
if (_buffer[_firstTokensInd] == null)
{
return next;
}
Token result = _buffer[_firstTokensInd];
_buffer[_firstTokensInd] = null;
if (_firstTokensInd != _lastTokenInd)
{
_firstTokensInd = IncTokenInd(_firstTokensInd);
}
return result;
}
protected void HandleNewLine() {
emit(PythonLexer.NEWLINE, HIDDEN, getText());
char next = (char) _input.LA(1);
// Process whitespaces in HandleSpaces
if (next != ' ' && next != '\t' && IsNotNewLineOrComment(next))
{
ProcessNewLine(0);
}
}
protected void HandleSpaces() {
char next = (char) _input.LA(1);
if ((_lastToken == null || _lastToken.getType() == PythonLexer.NEWLINE) && IsNotNewLineOrComment(next))
{
// Calculates the indentation of the provided spaces, taking the
// following rules into account:
//
// "Tabs are replaced (from left to right) by one to eight spaces
// such that the total number of characters up to and including
// the replacement is a multiple of eight [...]"
//
// -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
int indent = 0;
String text = getText();
for (int i = 0; i < text.length(); i++) {
indent += text.charAt(i) == '\t' ? TabSize - indent % TabSize : 1;
}
ProcessNewLine(indent);
}
emit(PythonLexer.WS, HIDDEN, getText());
}
protected void IncIndentLevel() {
_opened++;
}
protected void DecIndentLevel() {
if (_opened > 0) {
--_opened;
}
}
private boolean IsNotNewLineOrComment(char next) {
return _opened == 0 && next != '\r' && next != '\n' && next != '\f' && next != '#';
}
private void ProcessNewLine(int indent) {
emit(PythonLexer.LINE_BREAK);
int previous = _indents.size() == 0 ? 0 : _indents.peek();
if (indent > previous)
{
_indents.push(indent);
emit(PythonLexer.INDENT);
}
else
{
// Possibly emit more than 1 DEDENT token.
while (_indents.size() != 0 && _indents.peek() > indent)
{
emit(PythonLexer.DEDENT);
_indents.pop();
}
}
}
private int IncTokenInd(int ind) {
return (ind + 1) % _buffer.length;
}
private void emit(int tokenType) {
emit(tokenType, DEFAULT_TOKEN_CHANNEL, "");
}
private void emit(int tokenType, int channel, String text) {
int charIndex = getCharIndex();
CommonToken token = new CommonToken(_tokenFactorySourcePair, tokenType, channel, charIndex - text.length(), charIndex - 1);
token.setLine(getLine());
token.setCharPositionInLine(getCharPositionInLine());
token.setText(text);
emit(token);
}
}

26
codegpt-core/src/main/java/grammar/PythonParserBase.java
View file

@ -1,26 +0,0 @@
package grammar;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.TokenStream;
public abstract class PythonParserBase extends Parser
{
public PythonVersion Version = PythonVersion.Autodetect;
protected PythonParserBase(TokenStream input) {
super(input);
}
protected boolean CheckVersion(int version) {
return Version == PythonVersion.Autodetect || version == Version.getValue();
}
protected void SetVersion(int requiredVersion) {
if (requiredVersion == 2) {
Version = PythonVersion.Python2;
} else if (requiredVersion == 3) {
Version = PythonVersion.Python3;
}
}
}

17
codegpt-core/src/main/java/grammar/PythonVersion.java
View file

@ -1,17 +0,0 @@
package grammar;
public enum PythonVersion {
Autodetect(0),
Python2(2),
Python3(3);
private final int value;
PythonVersion(int value) {
this.value = value;
}
public int getValue() {
return value;
}
}

166
codegpt-core/src/main/java/grammar/TypeScriptLexerBase.java
View file

@ -1,166 +0,0 @@
package grammar;
import java.util.ArrayDeque;
import java.util.Deque;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
/**
* All lexer methods that used in grammar (IsStrictMode)
* should start with Upper Case Char similar to Lexer rules.
*/
public abstract class TypeScriptLexerBase extends Lexer
{
/**
* Stores values of nested modes. By default mode is strict or
* defined externally (useStrictDefault)
*/
private final Deque<Boolean> scopeStrictModes = new ArrayDeque<>();
private Token lastToken = null;
/**
* Default value of strict mode
* Can be defined externally by setUseStrictDefault
*/
private boolean useStrictDefault = false;
/**
* Current value of strict mode
* Can be defined during parsing, see StringFunctions.js and StringGlobal.js samples
*/
private boolean useStrictCurrent = false;
/**
* Keeps track of the current depth of nested template string backticks.
* E.g. after the X in:
*
* `${a ? `${X
*
* templateDepth will be 2. This variable is needed to determine if a `}` is a
* plain CloseBrace, or one that closes an expression inside a template string.
*/
private int templateDepth = 0;
/**
* Keeps track of the depth of open- and close-braces. Used for expressions like:
*
* `${[1, 2, 3].map(x => { return x * 2;}).join("")}`
*
* where the '}' from `return x * 2;}` should not become a `TemplateCloseBrace`
* token but rather a `CloseBrace` token.
*/
private int bracesDepth = 0;
public TypeScriptLexerBase(CharStream input) {
super(input);
}
public boolean getStrictDefault() {
return useStrictDefault;
}
public void setUseStrictDefault(boolean value) {
useStrictDefault = value;
useStrictCurrent = value;
}
public boolean IsStrictMode() {
return useStrictCurrent;
}
public void StartTemplateString() {
this.bracesDepth = 0;
}
public boolean IsInTemplateString() {
return this.templateDepth > 0 && this.bracesDepth == 0;
}
/**
* Return the next token from the character stream and records this last
* token in case it resides on the default channel. This recorded token
* is used to determine when the lexer could possibly match a regex
* literal. Also changes scopeStrictModes stack if tokenize special
* string 'use strict';
*
* @return the next token from the character stream.
*/
@Override
public Token nextToken() {
Token next = super.nextToken();
if (next.getChannel() == Token.DEFAULT_CHANNEL) {
// Keep track of the last token on the default channel.
this.lastToken = next;
}
return next;
}
protected void ProcessOpenBrace()
{
bracesDepth++;
useStrictCurrent = scopeStrictModes.size() > 0 && scopeStrictModes.peek() ? true : useStrictDefault;
scopeStrictModes.push(useStrictCurrent);
}
protected void ProcessCloseBrace()
{
bracesDepth--;
useStrictCurrent = scopeStrictModes.size() > 0 ? scopeStrictModes.pop() : useStrictDefault;
}
protected void ProcessStringLiteral()
{
if (lastToken == null || lastToken.getType() == TypeScriptLexer.OpenBrace)
{
String text = getText();
if (text.equals("\"use strict\"") || text.equals("'use strict'"))
{
if (scopeStrictModes.size() > 0)
scopeStrictModes.pop();
useStrictCurrent = true;
scopeStrictModes.push(useStrictCurrent);
}
}
}
protected void IncreaseTemplateDepth() {
this.templateDepth++;
}
protected void DecreaseTemplateDepth() {
this.templateDepth--;
}
/**
* Returns {@code true} if the lexer can match a regex literal.
*/
protected boolean IsRegexPossible() {
if (this.lastToken == null) {
// No token has been produced yet: at the start of the input,
// no division is possible, so a regex literal _is_ possible.
return true;
}
switch (this.lastToken.getType()) {
case TypeScriptLexer.Identifier:
case TypeScriptLexer.NullLiteral:
case TypeScriptLexer.BooleanLiteral:
case TypeScriptLexer.This:
case TypeScriptLexer.CloseBracket:
case TypeScriptLexer.CloseParen:
case TypeScriptLexer.OctalIntegerLiteral:
case TypeScriptLexer.DecimalLiteral:
case TypeScriptLexer.HexIntegerLiteral:
case TypeScriptLexer.StringLiteral:
case TypeScriptLexer.PlusPlus:
case TypeScriptLexer.MinusMinus:
// After any of the tokens above, no regex literal can follow.
return false;
default:
// In all other cases, a regex literal _is_ possible.
return true;
}
}
}

124
codegpt-core/src/main/java/grammar/TypeScriptParserBase.java
View file

@ -1,124 +0,0 @@
package grammar;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
/**
* All parser methods that used in grammar (p, prev, notLineTerminator, etc.)
* should start with lower case char similar to parser rules.
*/
public abstract class TypeScriptParserBase extends Parser
{
public TypeScriptParserBase(TokenStream input) {
super(input);
}
/**
* Short form for prev(String str)
*/
protected boolean p(String str) {
return prev(str);
}
/**
* Whether the previous token value equals to @param str
*/
protected boolean prev(String str) {
return _input.LT(-1).getText().equals(str);
}
/**
* Short form for next(String str)
*/
protected boolean n(String str) {
return next(str);
}
/**
* Whether the next token value equals to @param str
*/
protected boolean next(String str) {
return _input.LT(1).getText().equals(str);
}
protected boolean notLineTerminator() {
return !here(TypeScriptParser.LineTerminator);
}
protected boolean notOpenBraceAndNotFunction() {
int nextTokenType = _input.LT(1).getType();
return nextTokenType != TypeScriptParser.OpenBrace && nextTokenType != TypeScriptParser.Function_;
}
protected boolean closeBrace() {
return _input.LT(1).getType() == TypeScriptParser.CloseBrace;
}
/**
* Returns {@code true} iff on the current index of the parser's
* token stream a token of the given {@code type} exists on the
* {@code HIDDEN} channel.
*
* @param type
* the type of the token on the {@code HIDDEN} channel
* to check.
*
* @return {@code true} iff on the current index of the parser's
* token stream a token of the given {@code type} exists on the
* {@code HIDDEN} channel.
*/
private boolean here(final int type) {
// Get the token ahead of the current index.
int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1;
Token ahead = _input.get(possibleIndexEosToken);
// Check if the token resides on the HIDDEN channel and if it's of the
// provided type.
return (ahead.getChannel() == Lexer.HIDDEN) && (ahead.getType() == type);
}
/**
* Returns {@code true} iff on the current index of the parser's
* token stream a token exists on the {@code HIDDEN} channel which
* either is a line terminator, or is a multi line comment that
* contains a line terminator.
*
* @return {@code true} iff on the current index of the parser's
* token stream a token exists on the {@code HIDDEN} channel which
* either is a line terminator, or is a multi line comment that
* contains a line terminator.
*/
protected boolean lineTerminatorAhead() {
// Get the token ahead of the current index.
int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1;
Token ahead = _input.get(possibleIndexEosToken);
if (ahead.getChannel() != Lexer.HIDDEN) {
// We're only interested in tokens on the HIDDEN channel.
return false;
}
if (ahead.getType() == TypeScriptParser.LineTerminator) {
// There is definitely a line terminator ahead.
return true;
}
if (ahead.getType() == TypeScriptParser.WhiteSpaces) {
// Get the token ahead of the current whitespaces.
possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 2;
ahead = _input.get(possibleIndexEosToken);
}
// Get the token's text and type.
String text = ahead.getText();
int type = ahead.getType();
// Check if the token is, or contains a line terminator.
return (type == TypeScriptParser.MultiLineComment && (text.contains("\r") || text.contains("\n"))) ||
(type == TypeScriptParser.LineTerminator);
}
}

1
codegpt-core/src/main/resources/prompts/inline-completion-prompt.txt
View file

@ -1 +0,0 @@
{pre}{codeBefore}{suf}{codeAfter}{mid}

10
codegpt-core/src/main/resources/prompts/prompt-with-context.txt
View file

@ -1,10 +0,0 @@
Use the following pieces of context to answer the question at the end.
If you don't know the answer, just say that you don't know, don't try to make up an answer.
Context:
{context}
Question: {prompt}
Helpful answer in Markdown format:

10
codegpt-core/src/main/resources/prompts/text-generator.txt
View file

@ -1,10 +0,0 @@
You are Text Generator, a helpful expert of generating natural language into semantically comparable search query.
Text: List all the dependencies that the project uses
AI: project dependencies, development dependencies, versions, libraries, frameworks, packages
Text: Are there any scheduled tasks or background jobs running in our codebase, and if so, what are they responsible for?
AI: scheduled tasks, background jobs, cron jobs, task schedules, codebase tasks
Text: {prompt}
AI:

BIN
codegpt-core/src/test/resources/indexes/hnsw.index
View file

Binary file not shown.

3
settings.gradle.kts
View file

@ -1,4 +1,3 @@
rootProject.name = "CodeGPT"
include("codegpt-core")
include("codegpt-treesitter")
include(":codegpt-treesitter")
include(":codegpt-telemetry")

1
src/main/java/ee/carlrobert/codegpt/CodeGPTKeys.java
View file

@ -3,7 +3,6 @@ package ee.carlrobert.codegpt;
import com.intellij.openapi.editor.EditorCustomElementRenderer;
import com.intellij.openapi.editor.Inlay;
import com.intellij.openapi.util.Key;
import ee.carlrobert.embedding.ReferencedFile;
import java.util.List;
public class CodeGPTKeys {

1
src/main/java/ee/carlrobert/codegpt/ProjectCompilationStatusListener.java
View file

@ -18,7 +18,6 @@ import ee.carlrobert.codegpt.conversations.message.Message;
import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowContentManager;
import ee.carlrobert.codegpt.ui.OverlayUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;

2
codegpt-core/src/main/java/ee/carlrobert/embedding/ReferencedFile.java → src/main/java/ee/carlrobert/codegpt/ReferencedFile.java
View file

@ -1,4 +1,4 @@
package ee.carlrobert.embedding;
package ee.carlrobert.codegpt;
import java.io.File;
import java.io.IOException;

2
src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextAction.java
View file

@ -26,13 +26,13 @@ import com.intellij.util.ui.UI.PanelFactory;
import ee.carlrobert.codegpt.CodeGPTBundle;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.EncodingManager;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.settings.IncludedFilesSettings;
import ee.carlrobert.codegpt.ui.UIUtil;
import ee.carlrobert.codegpt.ui.checkbox.FileCheckboxTree;
import ee.carlrobert.codegpt.ui.checkbox.PsiElementCheckboxTree;
import ee.carlrobert.codegpt.ui.checkbox.VirtualFileCheckboxTree;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.Dimension;
import java.io.IOException;
import java.nio.file.Files;

2
src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextNotifier.java
View file

@ -1,7 +1,7 @@
package ee.carlrobert.codegpt.actions;
import com.intellij.util.messages.Topic;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.codegpt.ReferencedFile;
import java.util.List;
public interface IncludeFilesInContextNotifier {

2
src/main/java/ee/carlrobert/codegpt/actions/editor/EditorActionsUtil.java
View file

@ -11,11 +11,11 @@ import com.intellij.openapi.editor.impl.EditorImpl;
import com.intellij.openapi.extensions.PluginId;
import com.intellij.openapi.project.Project;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.conversations.message.Message;
import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowContentManager;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;

8
src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestHandler.java
View file

@ -17,15 +17,11 @@ public class CompletionRequestHandler {
private static final Logger LOG = Logger.getInstance(CompletionRequestHandler.class);
private final StringBuilder messageBuilder = new StringBuilder();
private final boolean useContextualSearch;
private final CompletionResponseEventListener completionResponseEventListener;
private SwingWorker<Void, String> swingWorker;
private EventSource eventSource;
public CompletionRequestHandler(
boolean useContextualSearch,
CompletionResponseEventListener completionResponseEventListener) {
this.useContextualSearch = useContextualSearch;
public CompletionRequestHandler(CompletionResponseEventListener completionResponseEventListener) {
this.completionResponseEventListener = completionResponseEventListener;
}
@ -46,7 +42,7 @@ public class CompletionRequestHandler {
CompletionEventListener<String> eventListener) {
try {
return CompletionRequestService.getInstance()
.getChatCompletionAsync(callParameters, useContextualSearch, eventListener);
.getChatCompletionAsync(callParameters, eventListener);
} catch (Throwable ex) {
handleCallException(ex);
throw ex;

63
src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestProvider.java
View file

@ -9,8 +9,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import ee.carlrobert.codegpt.CodeGPTPlugin;
import ee.carlrobert.codegpt.EncodingManager;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.completions.llama.LlamaModel;
import ee.carlrobert.codegpt.completions.llama.PromptTemplate;
import ee.carlrobert.codegpt.conversations.Conversation;
@ -29,8 +29,6 @@ import ee.carlrobert.codegpt.settings.service.openai.OpenAISettings;
import ee.carlrobert.codegpt.settings.service.you.YouSettings;
import ee.carlrobert.codegpt.telemetry.core.configuration.TelemetryConfiguration;
import ee.carlrobert.codegpt.telemetry.core.service.UserId;
import ee.carlrobert.embedding.EmbeddingsService;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.llm.client.anthropic.completion.ClaudeCompletionRequest;
import ee.carlrobert.llm.client.anthropic.completion.ClaudeCompletionRequestMessage;
import ee.carlrobert.llm.client.llama.completion.LlamaCompletionRequest;
@ -66,18 +64,14 @@ public class CompletionRequestProvider {
"/prompts/fix-compile-errors.txt");
private final EncodingManager encodingManager = EncodingManager.getInstance();
private final EmbeddingsService embeddingsService;
private final Conversation conversation;
public CompletionRequestProvider(Conversation conversation) {
this.embeddingsService = new EmbeddingsService(
CompletionClientProvider.getOpenAIClient(),
CodeGPTPlugin.getPluginBasePath());
this.conversation = conversation;
}
public static String getPromptWithContext(List<ReferencedFile> referencedFiles,
String userPrompt) {
String userPrompt) {
var includedFilesSettings = IncludedFilesSettings.getCurrentState();
var repeatableContext = referencedFiles.stream()
.map(item -> includedFilesSettings.getRepeatableContext()
@ -184,11 +178,10 @@ public class CompletionRequestProvider {
public OpenAIChatCompletionRequest buildOpenAIChatCompletionRequest(
@Nullable String model,
CallParameters callParameters,
boolean useContextualSearch,
@Nullable String overriddenPath) {
var configuration = ConfigurationSettings.getCurrentState();
var builder = new OpenAIChatCompletionRequest.Builder(
buildMessages(model, callParameters, useContextualSearch))
buildMessages(model, callParameters))
.setModel(model)
.setMaxTokens(configuration.getMaxTokens())
.setStream(true)
@ -206,7 +199,7 @@ public class CompletionRequestProvider {
CallParameters callParameters) {
return buildCustomOpenAIChatCompletionRequest(
customConfiguration,
buildMessages(callParameters, false),
buildMessages(callParameters),
true);
}
@ -273,43 +266,33 @@ public class CompletionRequestProvider {
return request;
}
private List<OpenAIChatCompletionMessage> buildMessages(
CallParameters callParameters,
boolean useContextualSearch) {
private List<OpenAIChatCompletionMessage> buildMessages(CallParameters callParameters) {
var message = callParameters.getMessage();
var messages = new ArrayList<OpenAIChatCompletionMessage>();
if (useContextualSearch) {
var prompt = embeddingsService.buildPromptWithContext(
message.getPrompt());
LOG.info("Retrieved context:\n" + prompt);
messages.add(new OpenAIChatCompletionMessage("user", prompt));
} else {
if (callParameters.getConversationType() == ConversationType.DEFAULT) {
messages.add(new OpenAIChatCompletionMessage(
"system",
ConfigurationSettings.getCurrentState().getSystemPrompt()));
}
if (callParameters.getConversationType() == ConversationType.FIX_COMPILE_ERRORS) {
messages.add(new OpenAIChatCompletionMessage("system", FIX_COMPILE_ERRORS_SYSTEM_PROMPT));
}
for (var prevMessage : conversation.getMessages()) {
if (callParameters.isRetry() && prevMessage.getId().equals(message.getId())) {
break;
}
messages.add(new OpenAIChatCompletionMessage("user", prevMessage.getPrompt()));
messages.add(new OpenAIChatCompletionMessage("assistant", prevMessage.getResponse()));
}
messages.add(new OpenAIChatCompletionMessage("user", message.getPrompt()));
if (callParameters.getConversationType() == ConversationType.DEFAULT) {
messages.add(new OpenAIChatCompletionMessage(
"system",
ConfigurationSettings.getCurrentState().getSystemPrompt()));
}
if (callParameters.getConversationType() == ConversationType.FIX_COMPILE_ERRORS) {
messages.add(new OpenAIChatCompletionMessage("system", FIX_COMPILE_ERRORS_SYSTEM_PROMPT));
}
for (var prevMessage : conversation.getMessages()) {
if (callParameters.isRetry() && prevMessage.getId().equals(message.getId())) {
break;
}
messages.add(new OpenAIChatCompletionMessage("user", prevMessage.getPrompt()));
messages.add(new OpenAIChatCompletionMessage("assistant", prevMessage.getResponse()));
}
messages.add(new OpenAIChatCompletionMessage("user", message.getPrompt()));
return messages;
}
private List<OpenAIChatCompletionMessage> buildMessages(
@Nullable String model,
CallParameters callParameters,
boolean useContextualSearch) {
var messages = buildMessages(callParameters, useContextualSearch);
CallParameters callParameters) {
var messages = buildMessages(callParameters);
if (model == null
|| GeneralSettings.getCurrentState().getSelectedService() == ServiceType.YOU) {

68
src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestService.java
View file

@ -61,68 +61,62 @@ public final class CompletionRequestService {
public EventSource getChatCompletionAsync(
CallParameters callParameters,
boolean useContextualSearch,
CompletionEventListener<String> eventListener) {
var requestProvider = new CompletionRequestProvider(callParameters.getConversation());
switch (GeneralSettings.getCurrentState().getSelectedService()) {
case OPENAI:
return switch (GeneralSettings.getCurrentState().getSelectedService()) {
case OPENAI -> {
var openAISettings = OpenAISettings.getCurrentState();
return CompletionClientProvider.getOpenAIClient().getChatCompletionAsync(
yield CompletionClientProvider.getOpenAIClient().getChatCompletionAsync(
requestProvider.buildOpenAIChatCompletionRequest(
openAISettings.getModel(),
callParameters,
useContextualSearch,
null),
eventListener);
case CUSTOM_OPENAI:
}
case CUSTOM_OPENAI -> {
var customConfiguration = CustomServiceSettings.getCurrentState();
return getCustomOpenAIChatCompletionAsync(
yield getCustomOpenAIChatCompletionAsync(
requestProvider.buildCustomOpenAIChatCompletionRequest(
customConfiguration,
callParameters),
eventListener);
case ANTHROPIC:
return CompletionClientProvider.getClaudeClient().getCompletionAsync(
requestProvider.buildAnthropicChatCompletionRequest(callParameters),
eventListener);
case AZURE:
}
case ANTHROPIC -> CompletionClientProvider.getClaudeClient().getCompletionAsync(
requestProvider.buildAnthropicChatCompletionRequest(callParameters),
eventListener);
case AZURE -> {
var azureSettings = AzureSettings.getCurrentState();
return CompletionClientProvider.getAzureClient().getChatCompletionAsync(
yield CompletionClientProvider.getAzureClient().getChatCompletionAsync(
requestProvider.buildOpenAIChatCompletionRequest(
null,
callParameters,
useContextualSearch,
azureSettings.isUsingCustomPath() ? azureSettings.getPath() : null),
eventListener);
case YOU:
return CompletionClientProvider.getYouClient().getChatCompletionAsync(
requestProvider.buildYouCompletionRequest(callParameters.getMessage()),
eventListener);
case LLAMA_CPP:
return CompletionClientProvider.getLlamaClient().getChatCompletionAsync(
requestProvider.buildLlamaCompletionRequest(
callParameters.getMessage(),
callParameters.getConversationType()),
eventListener);
default:
throw new IllegalArgumentException();
}
}
case YOU -> CompletionClientProvider.getYouClient().getChatCompletionAsync(
requestProvider.buildYouCompletionRequest(callParameters.getMessage()),
eventListener);
case LLAMA_CPP -> CompletionClientProvider.getLlamaClient().getChatCompletionAsync(
requestProvider.buildLlamaCompletionRequest(
callParameters.getMessage(),
callParameters.getConversationType()),
eventListener);
default -> throw new IllegalArgumentException();
};
}
public EventSource getCodeCompletionAsync(
InfillRequestDetails requestDetails,
CompletionEventListener<String> eventListener) {
var requestProvider = new CodeCompletionRequestProvider(requestDetails);
switch (GeneralSettings.getCurrentState().getSelectedService()) {
case OPENAI:
return CompletionClientProvider.getOpenAIClient()
.getCompletionAsync(requestProvider.buildOpenAIRequest(), eventListener);
case LLAMA_CPP:
return CompletionClientProvider.getLlamaClient()
.getChatCompletionAsync(requestProvider.buildLlamaRequest(), eventListener);
default:
throw new IllegalArgumentException("Code completion not supported for selected service");
}
return switch (GeneralSettings.getCurrentState().getSelectedService()) {
case OPENAI -> CompletionClientProvider.getOpenAIClient()
.getCompletionAsync(requestProvider.buildOpenAIRequest(), eventListener);
case LLAMA_CPP -> CompletionClientProvider.getLlamaClient()
.getChatCompletionAsync(requestProvider.buildLlamaRequest(), eventListener);
default ->
throw new IllegalArgumentException("Code completion not supported for selected service");
};
}
public void generateCommitMessageAsync(

28
src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingAction.java
View file

@ -1,28 +0,0 @@
package ee.carlrobert.codegpt.indexes;
import static com.intellij.openapi.ui.DialogWrapper.OK_EXIT_CODE;
import com.intellij.icons.AllIcons;
import com.intellij.openapi.actionSystem.AnAction;
import com.intellij.openapi.actionSystem.AnActionEvent;
import ee.carlrobert.codegpt.ui.OverlayUtil;
import org.jetbrains.annotations.NotNull;
public class CodebaseIndexingAction extends AnAction {
public CodebaseIndexingAction() {
super("Update Indexes", "Update indexes", AllIcons.Actions.Refresh);
}
@Override
public void actionPerformed(@NotNull AnActionEvent event) {
var project = event.getProject();
if (project != null) {
var folderStructureTreePanel = new FolderStructureTreePanel(project);
var show = OverlayUtil.showFileStructureDialog(project, folderStructureTreePanel);
if (show == OK_EXIT_CODE) {
new CodebaseIndexingTask(project, folderStructureTreePanel.getReferencedFiles()).run();
}
}
}
}

11
src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingCompletedNotifier.java
View file

@ -1,11 +0,0 @@
package ee.carlrobert.codegpt.indexes;
import com.intellij.util.messages.Topic;
public interface CodebaseIndexingCompletedNotifier {
Topic<CodebaseIndexingCompletedNotifier> INDEXING_COMPLETED_TOPIC =
Topic.create("codebaseIndexingCompleted", CodebaseIndexingCompletedNotifier.class);
void indexingCompleted();
}

81
src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingTask.java
View file

@ -1,81 +0,0 @@
package ee.carlrobert.codegpt.indexes;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.jelmerk.knn.Item;
import com.intellij.notification.NotificationType;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.Task;
import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator;
import com.intellij.openapi.project.Project;
import ee.carlrobert.codegpt.CodeGPTBundle;
import ee.carlrobert.codegpt.CodeGPTPlugin;
import ee.carlrobert.codegpt.completions.CompletionClientProvider;
import ee.carlrobert.codegpt.ui.OverlayUtil;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.EmbeddingsService;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.vector.VectorStore;
import java.util.List;
import java.util.Map;
import org.jetbrains.annotations.NotNull;
public class CodebaseIndexingTask extends Task.Backgroundable {
private static final Logger LOG = Logger.getInstance(CodebaseIndexingTask.class);
private final Project project;
private final List<ReferencedFile> referencedFiles;
private final EmbeddingsService embeddingsService;
public CodebaseIndexingTask(Project project, List<ReferencedFile> referencedFiles) {
super(project, CodeGPTBundle.get("codebaseIndexing.task.title"), true);
this.project = project;
this.referencedFiles = referencedFiles;
this.embeddingsService = new EmbeddingsService(
CompletionClientProvider.getOpenAIClient(),
CodeGPTPlugin.getPluginBasePath());
}
public void run() {
ProgressManager.getInstance()
.runProcessWithProgressAsynchronously(this, new BackgroundableProcessIndicator(this));
}
@Override
public void run(@NotNull ProgressIndicator indicator) {
LOG.info("Indexing started");
String fileContent;
try {
fileContent = new ObjectMapper().writeValueAsString(Map.of("content", referencedFiles));
} catch (JsonProcessingException e) {
throw new RuntimeException("Unable to serialize json file");
}
if (!com.intellij.openapi.util.io.FileUtil.exists(CodeGPTPlugin.getIndexStorePath())) {
FileUtil.tryCreateDirectory(CodeGPTPlugin.getIndexStorePath());
}
FileUtil.createFile(
CodeGPTPlugin.getProjectIndexStorePath(project), "index.json", fileContent);
try {
indicator.setFraction(0);
List<Item<Object, double[]>> embeddings =
embeddingsService.createEmbeddings(referencedFiles, indicator);
VectorStore.getInstance(CodeGPTPlugin.getPluginBasePath()).save(embeddings);
OverlayUtil.showNotification("Indexing completed", NotificationType.INFORMATION);
project.getMessageBus()
.syncPublisher(CodebaseIndexingCompletedNotifier.INDEXING_COMPLETED_TOPIC)
.indexingCompleted();
} catch (RuntimeException e) {
LOG.error("Something went wrong while indexing the codebase", e);
} finally {
if (indicator.isRunning()) {
indicator.stop();
}
}
}
}

242
src/main/java/ee/carlrobert/codegpt/indexes/FolderStructureTreePanel.java
View file

@ -1,242 +0,0 @@
package ee.carlrobert.codegpt.indexes;
import static java.util.stream.Collectors.toList;
import com.intellij.icons.AllIcons;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectUtil;
import com.intellij.openapi.vcs.changes.ChangeListManager;
import com.intellij.openapi.vcs.changes.VcsIgnoreManager;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl;
import com.intellij.openapi.vfs.newvfs.impl.VirtualFileImpl;
import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry;
import com.intellij.ui.CheckboxTree;
import com.intellij.ui.CheckboxTreeListener;
import com.intellij.ui.CheckedTreeNode;
import com.intellij.ui.ScrollPaneFactory;
import com.intellij.ui.components.JBLabel;
import com.intellij.util.ui.AsyncProcessIcon;
import com.intellij.util.ui.JBUI;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.BorderLayout;
import java.awt.FlowLayout;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.io.File;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import javax.swing.JPanel;
import javax.swing.JTree;
import javax.swing.SwingUtilities;
import org.jetbrains.annotations.NotNull;
public class FolderStructureTreePanel {
private CheckboxTree checkboxTree;
private final AsyncProcessIcon loadingFilesSpinner;
private final JPanel rootPanelContainer;
private final VcsIgnoreManager ignoreManager;
private final ChangeListManager changeListManager;
private long totalSize = 0;
public FolderStructureTreePanel(@NotNull Project project) {
this.ignoreManager = VcsIgnoreManager.getInstance(project);
this.loadingFilesSpinner = new AsyncProcessIcon("loading_files");
this.changeListManager = ChangeListManager.getInstance(project);
var projectDirectory = ProjectUtil.guessProjectDir(project);
if (projectDirectory == null) {
throw new RuntimeException("Couldn't find project directory");
}
var rootNode = new CheckedTreeNode(projectDirectory);
rootPanelContainer = new JPanel(new BorderLayout());
CompletableFuture
.runAsync(() -> traverseDirectory(rootNode, projectDirectory))
.thenRun(() -> SwingUtilities.invokeLater(() -> {
checkboxTree = createCheckboxTree(rootNode, true);
loadingFilesSpinner.setVisible(false);
updatePanel();
}));
checkboxTree = createCheckboxTree(rootNode, false);
rootPanelContainer.add(createRootPanel());
rootPanelContainer.add(createFooterPanel(), BorderLayout.SOUTH);
}
private CheckboxTree createCheckboxTree(CheckedTreeNode rootNode, boolean enabled) {
checkboxTree = new CheckboxTree(createFileTypesRenderer(), rootNode);
checkboxTree.setEditable(enabled);
checkboxTree.setEnabled(enabled);
checkboxTree.addCheckboxTreeListener(new CheckboxTreeListener() {
@Override
public void nodeStateChanged(@NotNull CheckedTreeNode node) {
try {
var length = ((VirtualFileImpl) node.getUserObject()).getLength();
if (node.isChecked()) {
totalSize += length;
} else {
totalSize -= length;
}
} catch (Throwable ignored) {
// ignore
}
}
});
checkboxTree.addMouseListener(new MouseAdapter() {
@Override
public void mouseClicked(MouseEvent e) {
updatePanel();
}
});
return checkboxTree;
}
private void updatePanel() {
rootPanelContainer.removeAll();
rootPanelContainer.add(createRootPanel());
rootPanelContainer.add(createFooterPanel(), BorderLayout.SOUTH);
rootPanelContainer.revalidate();
rootPanelContainer.repaint();
}
public JPanel getPanel() {
return rootPanelContainer;
}
private JPanel createRootPanel() {
var scrollPane = ScrollPaneFactory.createScrollPane(checkboxTree);
scrollPane.setPreferredSize(JBUI.size(250, 375));
return JBUI.Panels.simplePanel().addToCenter(scrollPane);
}
private JPanel createFooterPanel() {
var panel = new JPanel(new FlowLayout(FlowLayout.LEFT, 0, 8));
if (loadingFilesSpinner.isVisible()) {
panel.add(new JBLabel("Total size:"));
panel.add(loadingFilesSpinner);
} else {
panel.add(new JBLabel("Total size: "
+ FileUtil.convertFileSize(totalSize) + " ~ "
+ (FileUtil.convertLongValue(totalSize / 4)) + " tokens " + " ~ "
+ new DecimalFormat("#.##")
.format(((double) (totalSize / 4) / 1000) * 0.0001) + " $"));
}
return panel;
}
public List<ReferencedFile> getReferencedFiles() {
return getCheckedVirtualFiles().stream()
.map(item -> new ReferencedFile(new File(item.getPath())))
.collect(toList());
}
private List<VirtualFileImpl> getCheckedVirtualFiles() {
return Arrays.stream(checkboxTree.getCheckedNodes(
VirtualFileSystemEntry.class,
node -> node instanceof VirtualFileImpl))
.map(entry -> (VirtualFileImpl) entry)
.collect(toList());
}
private void processChildFile(@NotNull CheckedTreeNode node, @NotNull VirtualFile file) {
long fileSize = file.getLength();
try {
if (node.isChecked()) {
node.setChecked(!changeListManager.isIgnoredFile(file)
&& !ignoreManager.isPotentiallyIgnoredFile(file)
&& FileUtil.isUtf8File(file.getPath())
&& fileSize < Math.pow(1024, 2));
}
if (node.isChecked()) {
totalSize += fileSize;
}
} catch (RuntimeException ignored) {
// ignore
}
}
private void traverseDirectory(@NotNull CheckedTreeNode parentNode,
@NotNull VirtualFile projectDirectory) {
for (VirtualFile childFile : projectDirectory.getChildren()) {
var node = new CheckedTreeNode(childFile);
parentNode.add(node);
var potentiallyIgnored = ignoredFileDirectories.parallelStream()
.anyMatch(it -> it.equalsIgnoreCase(childFile.getName()));
if (!parentNode.isChecked() || potentiallyIgnored) {
node.setChecked(false);
}
if (childFile.isDirectory()) {
traverseDirectory(node, childFile);
} else {
processChildFile(node, childFile);
}
}
}
private @NotNull CheckboxTree.CheckboxTreeCellRenderer createFileTypesRenderer() {
return new CheckboxTree.CheckboxTreeCellRenderer() {
@Override
public void customizeRenderer(JTree t,
Object value,
boolean selected,
boolean expanded,
boolean leaf,
int row,
boolean focus) {
if (!(value instanceof CheckedTreeNode)) {
return;
}
var node = (CheckedTreeNode) value;
var userObject = node.getUserObject();
if (userObject instanceof VirtualFileSystemEntry) {
getTextRenderer().append(((VirtualFileSystemEntry) userObject).getName());
if (userObject instanceof VirtualDirectoryImpl) {
getTextRenderer().setIcon(AllIcons.Nodes.Folder);
} else {
var fileType = FileTypeManager.getInstance()
.getFileTypeByFile((VirtualFileSystemEntry) userObject);
getTextRenderer().setIcon(fileType.getIcon());
getTextRenderer().append(
" - " + FileUtil.convertFileSize(
((VirtualFileSystemEntry) userObject).getLength()));
}
}
}
};
}
private static final List<String> ignoredFileDirectories = List.of(
"node_modules",
".git",
".svn",
".bzr",
".cvs",
".m2",
".idea",
".vscode",
".project",
".settings",
"node_modules",
"vendor",
"lib",
"build",
"target",
"media",
"logs",
"uploads");
}

4
src/main/java/ee/carlrobert/codegpt/toolwindow/ProjectToolWindowFactory.java
View file

@ -6,7 +6,6 @@ import com.intellij.openapi.wm.ToolWindow;
import com.intellij.openapi.wm.ToolWindowFactory;
import com.intellij.ui.content.ContentManagerEvent;
import com.intellij.ui.content.ContentManagerListener;
import ee.carlrobert.codegpt.toolwindow.chat.contextual.ContextualChatToolWindowPanel;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowPanel;
import ee.carlrobert.codegpt.toolwindow.conversations.ConversationsToolWindow;
import javax.swing.JComponent;
@ -16,12 +15,9 @@ public class ProjectToolWindowFactory implements ToolWindowFactory, DumbAware {
public void createToolWindowContent(@NotNull Project project, @NotNull ToolWindow toolWindow) {
var chatToolWindowPanel = new StandardChatToolWindowPanel(project, toolWindow.getDisposable());
// var contextualChatToolWindowPanel =
// new ContextualChatToolWindowPanel(project, toolWindow.getDisposable());
var conversationsToolWindow = new ConversationsToolWindow(project);
addContent(toolWindow, chatToolWindowPanel, "Chat");
// addContent(toolWindow, contextualChatToolWindowPanel, "Contextual Chat");
addContent(toolWindow, conversationsToolWindow.getContent(), "Chat History");
toolWindow.addContentManagerListener(new ContentManagerListener() {
public void selectionChanged(@NotNull ContentManagerEvent event) {

5
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java
View file

@ -14,6 +14,7 @@ import com.intellij.ui.JBColor;
import com.intellij.util.ui.JBUI;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.EncodingManager;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.actions.ActionType;
import ee.carlrobert.codegpt.completions.CallParameters;
import ee.carlrobert.codegpt.completions.CompletionRequestHandler;
@ -37,7 +38,6 @@ import ee.carlrobert.codegpt.toolwindow.chat.ui.textarea.UserPromptTextArea;
import ee.carlrobert.codegpt.toolwindow.chat.ui.textarea.UserPromptTextAreaHeader;
import ee.carlrobert.codegpt.util.EditorUtil;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.BorderLayout;
import java.awt.GridBagConstraints;
import java.awt.GridBagLayout;
@ -51,7 +51,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
private static final Logger LOG = Logger.getInstance(ChatToolWindowTabPanel.class);
private final boolean useContextualSearch;
private final JPanel rootPanel;
private final Conversation conversation;
private final UserPromptTextArea userPromptTextArea;
@ -69,7 +68,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
boolean useContextualSearch) {
this.project = project;
this.conversation = conversation;
this.useContextualSearch = useContextualSearch;
conversationService = ConversationService.getInstance();
toolWindowScrollablePanel = new ChatToolWindowScrollablePanel();
totalTokensPanel = new TotalTokensPanel(
@ -217,7 +215,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
}
var requestHandler = new CompletionRequestHandler(
useContextualSearch,
new ToolWindowCompletionResponseEventListener(
conversationService,
responsePanel,

113
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowLandingPanel.java
View file

@ -1,113 +0,0 @@
package ee.carlrobert.codegpt.toolwindow.chat.contextual;
import static com.intellij.openapi.ui.DialogWrapper.OK_EXIT_CODE;
import static javax.swing.event.HyperlinkEvent.EventType.ACTIVATED;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.options.ShowSettingsUtil;
import com.intellij.openapi.project.Project;
import ee.carlrobert.codegpt.CodeGPTPlugin;
import ee.carlrobert.codegpt.indexes.CodebaseIndexingCompletedNotifier;
import ee.carlrobert.codegpt.indexes.CodebaseIndexingTask;
import ee.carlrobert.codegpt.indexes.FolderStructureTreePanel;
import ee.carlrobert.codegpt.settings.GeneralSettingsConfigurable;
import ee.carlrobert.codegpt.toolwindow.chat.ui.ResponsePanel;
import ee.carlrobert.codegpt.ui.OverlayUtil;
import ee.carlrobert.codegpt.ui.UIUtil;
import ee.carlrobert.vector.VectorStore;
import javax.swing.JTextPane;
import javax.swing.event.HyperlinkEvent;
class ContextualChatToolWindowLandingPanel extends ResponsePanel {
private static final Logger LOG = Logger.getInstance(ContextualChatToolWindowLandingPanel.class);
private final Project project;
private final EditorActionEvent actionEvent;
ContextualChatToolWindowLandingPanel(Project project, EditorActionEvent actionEvent) {
this.project = project;
this.actionEvent = actionEvent;
addContent(createContent());
project.getMessageBus()
.connect()
.subscribe(CodebaseIndexingCompletedNotifier.INDEXING_COMPLETED_TOPIC,
(CodebaseIndexingCompletedNotifier) () -> updateContent(createContent()));
}
private JTextPane createContent() {
var description = UIUtil.createTextPane("", false, this::handleHyperlinkClicked);
if (VectorStore.getInstance(CodeGPTPlugin.getPluginBasePath()).isIndexExists()) {
description.setText("<html>"
+ "<p style=\"margin-top: 4px; margin-bottom: 4px;\">"
+ "Feel free to ask me anything about your codebase, and I'll be your helpful guide, "
+ "dedicated to providing you with the best answers possible!"
+ "</p>"
+ "<p style=\"margin-top: 4px; margin-bottom: 4px;\">"
+ "Here are a few examples of how I might be helpful:"
+ "</p>"
+ "<ul>"
+ "<li>"
+ "<a href=\"LIST_DEPENDENCIES\">List all the dependencies that the project uses</a>"
+ "</li>"
+ "<li>"
+ "<a href=\"SCHEDULED_TASKS\">Are there any scheduled tasks or background jobs "
+ "running in our codebase, and if so, what are they responsible for?</a>"
+ "</li>"
+ "<li>"
+ "<a href=\"AUTHENTICATION_MECHANISM\">Can you provide an overview of the "
+ "authentication and authorization mechanism implemented in our application?</a>"
+ "</li>"
+ "</ul>"
+ "</html>");
} else {
description.setText("<html>"
+ "<p style=\"margin-top: 4px; margin-bottom: 4px;\">"
+ "It looks like you haven't indexed your codebase yet."
+ "</p>"
+ "<p style=\"margin-top: 4px; margin-bottom: 4px;\">"
+ "<a href=\"START_INDEXING\">Start indexing</a> your codebase to get "
+ "access to contextual chat experience."
+ "</p>"
+ "</html>");
}
return description;
}
private void handleHyperlinkClicked(HyperlinkEvent event) {
if (ACTIVATED.equals(event.getEventType())) {
if (event.getURL() == null) {
switch (event.getDescription()) {
case "LOGIN":
ShowSettingsUtil.getInstance()
.showSettingsDialog(project, GeneralSettingsConfigurable.class);
break;
case "LIST_DEPENDENCIES":
actionEvent.handleAction("List all the dependencies that the project uses");
break;
case "SCHEDULED_TASKS":
actionEvent.handleAction("Are there any scheduled tasks or background "
+ "jobs running in our codebase, and if so, what are they responsible for?");
break;
case "AUTHENTICATION_MECHANISM":
actionEvent.handleAction("Can you provide an overview of the authentication "
+ "and authorization mechanism implemented in our application?");
break;
case "START_INDEXING":
var folderStructureTreePanel = new FolderStructureTreePanel(project);
var show = OverlayUtil.showFileStructureDialog(project, folderStructureTreePanel);
if (show == OK_EXIT_CODE) {
new CodebaseIndexingTask(project, folderStructureTreePanel.getReferencedFiles())
.run();
}
break;
default:
LOG.error("Could not trigger action {}", event.getDescription());
}
} else {
UIUtil.handleHyperlinkClicked(event);
}
}
}
}

40
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowPanel.java
View file

@ -1,40 +0,0 @@
package ee.carlrobert.codegpt.toolwindow.chat.contextual;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.actionSystem.ActionManager;
import com.intellij.openapi.actionSystem.ActionToolbar;
import com.intellij.openapi.actionSystem.DefaultActionGroup;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.SimpleToolWindowPanel;
import com.intellij.openapi.util.Disposer;
import ee.carlrobert.codegpt.actions.toolwindow.ClearChatWindowAction;
import ee.carlrobert.codegpt.conversations.Conversation;
import ee.carlrobert.codegpt.indexes.CodebaseIndexingAction;
import org.jetbrains.annotations.NotNull;
public class ContextualChatToolWindowPanel extends SimpleToolWindowPanel {
public ContextualChatToolWindowPanel(
@NotNull Project project,
@NotNull Conversation conversation,
@NotNull Disposable parentDisposable) {
super(true);
var tabPanel = new ContextualChatToolWindowTabPanel(project, conversation);
setToolbar(createActionToolbar(tabPanel).getComponent());
setContent(tabPanel.getContent());
Disposer.register(parentDisposable, tabPanel);
}
private ActionToolbar createActionToolbar(ContextualChatToolWindowTabPanel tabPanel) {
var actionGroup = new DefaultActionGroup("TOOLBAR_ACTION_GROUP", false);
actionGroup.add(new ClearChatWindowAction(tabPanel::displayLandingView));
actionGroup.addSeparator();
actionGroup.add(new CodebaseIndexingAction());
var toolbar = ActionManager.getInstance()
.createActionToolbar("NAVIGATION_BAR_TOOLBAR", actionGroup, false);
toolbar.setTargetComponent(this);
return toolbar;
}
}

26
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowTabPanel.java
View file

@ -1,26 +0,0 @@
package ee.carlrobert.codegpt.toolwindow.chat.contextual;
import com.intellij.openapi.project.Project;
import ee.carlrobert.codegpt.completions.ConversationType;
import ee.carlrobert.codegpt.conversations.Conversation;
import ee.carlrobert.codegpt.conversations.message.Message;
import ee.carlrobert.codegpt.toolwindow.chat.ChatToolWindowTabPanel;
import javax.swing.JComponent;
import org.jetbrains.annotations.NotNull;
public class ContextualChatToolWindowTabPanel extends ChatToolWindowTabPanel {
public ContextualChatToolWindowTabPanel(
@NotNull Project project,
@NotNull Conversation conversation) {
super(project, conversation, true);
displayLandingView();
}
@Override
protected JComponent getLandingView() {
return new ContextualChatToolWindowLandingPanel(
project,
(prompt) -> sendMessage(new Message(prompt), ConversationType.DEFAULT));
}
}

7
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/EditorActionEvent.java
View file

@ -1,7 +0,0 @@
package ee.carlrobert.codegpt.toolwindow.chat.contextual;
@FunctionalInterface
public interface EditorActionEvent {
void handleAction(String prompt);
}

2
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/standard/StandardChatToolWindowPanel.java
View file

@ -8,6 +8,7 @@ import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.SimpleToolWindowPanel;
import com.intellij.openapi.util.Disposer;
import com.intellij.util.ui.JBUI;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier;
import ee.carlrobert.codegpt.actions.toolwindow.ClearChatWindowAction;
import ee.carlrobert.codegpt.actions.toolwindow.CreateNewConversationAction;
@ -15,7 +16,6 @@ import ee.carlrobert.codegpt.actions.toolwindow.OpenInEditorAction;
import ee.carlrobert.codegpt.conversations.ConversationService;
import ee.carlrobert.codegpt.conversations.ConversationsState;
import ee.carlrobert.codegpt.toolwindow.chat.ui.SelectedFilesNotification;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.BorderLayout;
import java.util.List;
import javax.swing.JPanel;

2
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/SelectedFilesNotification.java
View file

@ -11,8 +11,8 @@ import com.intellij.ui.components.JBLabel;
import com.intellij.util.ui.JBUI;
import com.intellij.util.ui.JBUI.CurrentTheme.NotificationInfo;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.BorderLayout;
import java.nio.file.Paths;
import java.util.List;

2
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/textarea/TotalTokensPanel.java
View file

@ -14,9 +14,9 @@ import com.intellij.ui.components.JBLabel;
import com.intellij.util.ui.JBUI;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.EncodingManager;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier;
import ee.carlrobert.codegpt.conversations.Conversation;
import ee.carlrobert.embedding.ReferencedFile;
import java.awt.FlowLayout;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;

29
src/main/java/ee/carlrobert/codegpt/ui/OverlayUtil.java
View file

@ -12,7 +12,6 @@ import com.intellij.notification.NotificationType;
import com.intellij.notification.Notifications;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.ui.DialogBuilder;
import com.intellij.openapi.ui.DoNotAskOption;
import com.intellij.openapi.ui.MessageDialogBuilder;
import com.intellij.openapi.ui.MessageType;
@ -21,12 +20,8 @@ import com.intellij.openapi.ui.popup.Balloon;
import com.intellij.openapi.ui.popup.Balloon.Position;
import com.intellij.openapi.ui.popup.JBPopupFactory;
import com.intellij.ui.awt.RelativePoint;
import com.intellij.ui.components.JBLabel;
import com.intellij.util.ui.JBFont;
import com.intellij.util.ui.JBUI;
import ee.carlrobert.codegpt.CodeGPTBundle;
import ee.carlrobert.codegpt.conversations.ConversationsState;
import ee.carlrobert.codegpt.indexes.FolderStructureTreePanel;
import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings;
import ee.carlrobert.codegpt.util.EditorUtil;
import java.awt.Point;
@ -44,30 +39,6 @@ public class OverlayUtil {
Notifications.Bus.notify(getDefaultNotification(content, type));
}
public static int showFileStructureDialog(
Project project,
FolderStructureTreePanel folderStructureTreePanel) {
var dialogBuilder = new DialogBuilder(project);
dialogBuilder.setNorthPanel(JBUI.Panels.simplePanel(new JBLabel(
"<html>"
+ "<p>Indexing files enables direct queries related to your codebase.</p>"
+ "<br/>"
+ "<p>File indexing occurs locally on your computer; "
+ "no files are sent to any 3rd party services.</p>"
+ "<p>For additional information, refer to the "
+ "<a href=\"https://google.com\">CodeGPT documentation</a>.</p>"
+ "</html>")
.setCopyable(true)
.setAllowAutoWrapping(true)
.withFont(JBFont.medium()))
.withBorder(JBUI.Borders.emptyBottom(12)));
dialogBuilder.setCenterPanel(folderStructureTreePanel.getPanel());
dialogBuilder.addOkAction().setText("Start Indexing");
dialogBuilder.addCancelAction();
dialogBuilder.setTitle("Choose Files for Indexing");
return dialogBuilder.show();
}
public static int showDeleteConversationDialog() {
return Messages.showYesNoDialog(
CodeGPTBundle.get("dialog.deleteConversation.description"),

2
src/main/java/ee/carlrobert/codegpt/ui/checkbox/FileCheckboxTree.java
View file

@ -5,8 +5,8 @@ import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.ui.CheckboxTree;
import com.intellij.ui.CheckedTreeNode;
import com.intellij.ui.ColoredTreeCellRenderer;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.util.file.FileUtil;
import ee.carlrobert.embedding.ReferencedFile;
import java.util.List;
import org.jetbrains.annotations.NotNull;

2
src/main/java/ee/carlrobert/codegpt/ui/checkbox/PsiElementCheckboxTree.java
View file

@ -9,7 +9,7 @@ import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.impl.file.PsiDirectoryImpl;
import com.intellij.ui.CheckedTreeNode;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.codegpt.ReferencedFile;
import java.io.File;
import java.util.Arrays;
import java.util.List;

5
src/main/java/ee/carlrobert/codegpt/ui/checkbox/VirtualFileCheckboxTree.java
View file

@ -6,7 +6,7 @@ import com.intellij.icons.AllIcons;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.ui.CheckedTreeNode;
import com.intellij.util.PlatformIcons;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.codegpt.ReferencedFile;
import java.io.File;
import java.util.Arrays;
import java.util.List;
@ -53,8 +53,7 @@ public class VirtualFileCheckboxTree extends FileCheckboxTree {
return new FileCheckboxTreeCellRenderer() {
@Override
void updatePresentation(Object userObject) {
if (userObject instanceof VirtualFile) {
VirtualFile virtualFile = (VirtualFile) userObject;
if (userObject instanceof VirtualFile virtualFile) {
if (virtualFile.isDirectory()) {
getTextRenderer().append(virtualFile.getName());
getTextRenderer().setIcon(PlatformIcons.FOLDER_ICON);

0
codegpt-core/src/main/resources/prompts/default-completion-system-prompt.txt → src/main/resources/prompts/default-completion-system-prompt.txt
View file

0
codegpt-core/src/main/resources/prompts/fix-compile-errors.txt → src/main/resources/prompts/fix-compile-errors.txt
View file

0
codegpt-core/src/main/resources/prompts/generate-commit-message-system-prompt.txt → src/main/resources/prompts/generate-commit-message-system-prompt.txt
View file

0
codegpt-core/src/main/resources/prompts/method-name-generator.txt → src/main/resources/prompts/method-name-generator.txt
View file

86
src/test/java/ee/carlrobert/codegpt/completions/CompletionRequestProviderTest.java
View file

@ -1,11 +1,6 @@
package ee.carlrobert.codegpt.completions;
import static ee.carlrobert.codegpt.completions.CompletionRequestProvider.COMPLETION_SYSTEM_PROMPT;
import static ee.carlrobert.llm.client.util.JSONUtil.e;
import static ee.carlrobert.llm.client.util.JSONUtil.jsonArray;
import static ee.carlrobert.llm.client.util.JSONUtil.jsonMap;
import static ee.carlrobert.llm.client.util.JSONUtil.jsonMapResponse;
import static org.apache.http.HttpHeaders.AUTHORIZATION;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.groups.Tuple.tuple;
@ -13,11 +8,7 @@ import ee.carlrobert.codegpt.conversations.ConversationService;
import ee.carlrobert.codegpt.conversations.message.Message;
import ee.carlrobert.codegpt.credentials.OpenAICredentialManager;
import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings;
import ee.carlrobert.llm.client.http.ResponseEntity;
import ee.carlrobert.llm.client.http.exchange.BasicHttpExchange;
import ee.carlrobert.llm.client.openai.completion.OpenAIChatCompletionModel;
import java.util.List;
import java.util.Map;
import testsupport.IntegrationTest;
public class CompletionRequestProviderTest extends IntegrationTest {
@ -39,7 +30,6 @@ public class CompletionRequestProviderTest extends IntegrationTest {
ConversationType.DEFAULT,
new Message("TEST_CHAT_COMPLETION_PROMPT"),
false),
false,
null);
assertThat(request.getMessages())
@ -68,7 +58,6 @@ public class CompletionRequestProviderTest extends IntegrationTest {
ConversationType.DEFAULT,
new Message("TEST_CHAT_COMPLETION_PROMPT"),
false),
false,
null);
assertThat(request.getMessages())
@ -98,7 +87,6 @@ public class CompletionRequestProviderTest extends IntegrationTest {
ConversationType.DEFAULT,
secondMessage,
true),
false,
null);
assertThat(request.getMessages())
@ -128,7 +116,6 @@ public class CompletionRequestProviderTest extends IntegrationTest {
ConversationType.DEFAULT,
new Message("TEST_CHAT_COMPLETION_PROMPT"),
false),
false,
null);
assertThat(request.getMessages())
@ -155,82 +142,9 @@ public class CompletionRequestProviderTest extends IntegrationTest {
ConversationType.DEFAULT,
createDummyMessage(100),
false),
false,
null));
}
public void testContextualSearch() {
useOpenAIService();
var conversation = ConversationService.getInstance().startConversation();
expectOpenAI((BasicHttpExchange) request -> {
assertThat(request.getUri().getPath()).isEqualTo("/v1/chat/completions");
assertThat(request.getMethod()).isEqualTo("POST");
assertThat(request.getHeaders().get(AUTHORIZATION).get(0)).isEqualTo("Bearer TEST_API_KEY");
assertThat(request.getBody())
.extracting("model", "messages")
.containsExactly("gpt-4",
List.of(Map.of(
"role", "user",
"content",
"You are Text Generator, a helpful expert of generating natural "
+ "language into semantically comparable search query.\n"
+ "\n"
+ "Text: List all the dependencies that the project uses\n"
+ "AI: project dependencies, development dependencies, "
+ "versions, libraries, frameworks, packages\n"
+ "\n"
+ "Text: Are there any scheduled tasks or background jobs "
+ "running in our codebase, and if so, what are they responsible for?\n"
+ "AI: scheduled tasks, background jobs, cron jobs, "
+ "task schedules, codebase tasks\n"
+ "\n"
+ "Text: TEST_CHAT_COMPLETION_PROMPT\n"
+ "AI:")));
return new ResponseEntity(200,
jsonMapResponse("choices", jsonArray(jsonMap("message",
jsonMap(e("role", "assistant"), e("content", "TEST_CHAT_COMPLETION_RESPONSE"))))));
});
expectOpenAI((BasicHttpExchange) request -> {
assertThat(request.getUri().getPath()).isEqualTo("/v1/embeddings");
var headers = request.getHeaders();
assertThat(headers.get("Authorization").get(0)).isEqualTo("Bearer TEST_API_KEY");
assertThat(request.getBody())
.extracting("model", "input")
.containsExactly("text-embedding-ada-002", List.of("TEST_CHAT_COMPLETION_RESPONSE"));
return new ResponseEntity(200, jsonMapResponse("data",
jsonArray(jsonMap("embedding", List.of(-0.00692, -0.0053, -4.5471, -0.0240)))));
});
var request = new CompletionRequestProvider(conversation)
.buildOpenAIChatCompletionRequest(
OpenAIChatCompletionModel.GPT_3_5.getCode(),
new CallParameters(
conversation,
ConversationType.DEFAULT,
new Message("TEST_CHAT_COMPLETION_PROMPT"),
false),
true,
null);
assertThat(request.getModel()).isEqualTo("gpt-3.5-turbo");
assertThat(request.getMessages().size()).isEqualTo(1);
assertThat(request.getMessages().get(0))
.extracting("role", "content")
.containsExactly("user",
"Use the following pieces of context to answer the question at the end.\n"
+ "If you don't know the answer, just say that you don't know, "
+ "don't try to make up an answer.\n"
+ "\n"
+ "Context:\n"
+ "\n"
+ "TEST_CONTEXT\n"
+ "\n"
+ "Question: TEST_CHAT_COMPLETION_PROMPT\n"
+ "\n"
+ "Helpful answer in Markdown format:");
}
private Message createDummyMessage(int tokenSize) {
return createDummyMessage("TEST_PROMPT", tokenSize);
}

10
src/test/java/ee/carlrobert/codegpt/completions/DefaultCompletionRequestHandlerTest.java
View file

@ -27,7 +27,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest {
useOpenAIService();
var message = new Message("TEST_PROMPT");
var conversation = ConversationService.getInstance().startConversation();
var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message));
var requestHandler = new CompletionRequestHandler(getRequestEventListener(message));
expectOpenAI((StreamHttpExchange) request -> {
assertThat(request.getUri().getPath()).isEqualTo("/v1/chat/completions");
assertThat(request.getMethod()).isEqualTo("POST");
@ -83,7 +83,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest {
jsonMapResponse("choices", jsonArray(jsonMap("delta", jsonMap("content", "!")))));
});
var message = new Message("TEST_PROMPT");
var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message));
var requestHandler = new CompletionRequestHandler(getRequestEventListener(message));
requestHandler.call(new CallParameters(conversation, ConversationType.DEFAULT, message, false));
@ -122,7 +122,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest {
jsonMapResponse("choices", jsonArray(jsonMap("delta", jsonMap("content", "!")))));
});
var message = new Message("TEST_PROMPT");
var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message));
var requestHandler = new CompletionRequestHandler(getRequestEventListener(message));
requestHandler.call(new CallParameters(conversation, ConversationType.DEFAULT, message, false));
@ -134,7 +134,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest {
var message = new Message("TEST_PROMPT");
var conversation = ConversationService.getInstance().startConversation();
conversation.addMessage(new Message("Ping", "Pong"));
var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message));
var requestHandler = new CompletionRequestHandler(getRequestEventListener(message));
expectYou((StreamHttpExchange) request -> {
assertThat(request.getUri().getPath()).isEqualTo("/api/streamingSearch");
assertThat(request.getMethod()).isEqualTo("GET");
@ -186,7 +186,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest {
var message = new Message("TEST_PROMPT");
var conversation = ConversationService.getInstance().startConversation();
conversation.addMessage(new Message("Ping", "Pong"));
var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message));
var requestHandler = new CompletionRequestHandler(getRequestEventListener(message));
expectLlama((StreamHttpExchange) request -> {
assertThat(request.getUri().getPath()).isEqualTo("/completion");
assertThat(request.getBody())

2
src/test/java/ee/carlrobert/codegpt/toolwindow/chat/StandardChatToolWindowTabPanelTest.java
View file

@ -14,6 +14,7 @@ import static org.awaitility.Awaitility.await;
import ee.carlrobert.codegpt.CodeGPTKeys;
import ee.carlrobert.codegpt.EncodingManager;
import ee.carlrobert.codegpt.ReferencedFile;
import ee.carlrobert.codegpt.completions.ConversationType;
import ee.carlrobert.codegpt.completions.HuggingFaceModel;
import ee.carlrobert.codegpt.conversations.ConversationService;
@ -21,7 +22,6 @@ import ee.carlrobert.codegpt.conversations.message.Message;
import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings;
import ee.carlrobert.codegpt.settings.service.llama.LlamaSettings;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowTabPanel;
import ee.carlrobert.embedding.ReferencedFile;
import ee.carlrobert.llm.client.http.exchange.StreamHttpExchange;
import java.util.List;
import java.util.Map;