Replace row/column based Location with byte-offsets.

Author: MichaReiser

Cargo.toml

@@ -12,6 +12,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 resolver = "2"
 members = [
     "ast", "core", "literal", "parser",
+    "ruff_text_size",
 ]
 
 [workspace.dependencies]

ast/Cargo.toml

@@ -16,5 +16,6 @@ unparse = ["rustpython-literal"]
 [dependencies]
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
+ruff_text_size =  { path = "../ruff_text_size" }
 
 num-bigint = { workspace = true }

ast/asdl_rs.py

@@ -238,13 +238,18 @@ def visitField(self, field, parent, vis, depth, constructor=None):
         if fieldtype and fieldtype.has_userdata:
             typ = f"{typ}<U>"
         # don't box if we're doing Vec<T>, but do box if we're doing Vec<Option<Box<T>>>
-        if fieldtype and fieldtype.boxed and (not (parent.product or field.seq) or field.opt):
+        if (
+            fieldtype
+            and fieldtype.boxed
+            and (not (parent.product or field.seq) or field.opt)
+        ):
             typ = f"Box<{typ}>"
         if field.opt or (
             # When a dictionary literal contains dictionary unpacking (e.g., `{**d}`),
             # the expression to be unpacked goes in `values` with a `None` at the corresponding
             # position in `keys`. To handle this, the type of `keys` needs to be `Option<Vec<T>>`.
-            constructor == "Dict" and field.name == "keys"
+            constructor == "Dict"
+            and field.name == "keys"
         ):
             typ = f"Option<{typ}>"
         if field.seq:
@@ -311,7 +316,7 @@ def visitModule(self, mod, depth):
             depth,
         )
         self.emit(
-            "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, end_location: node.end_location, node: f(folder, node.node)? })",
+            "Ok(Located { custom: folder.map_user(node.custom)?, range: node.range, node: f(folder, node.node)? })",
             depth + 1,
         )
         self.emit("}", depth)
@@ -649,7 +654,7 @@ def write_ast_def(mod, typeinfo, f):
         #![allow(clippy::derive_partial_eq_without_eq)]
         
         pub use crate::constant::*;
-        pub use crate::Location;
+        pub use ruff_text_size::{TextSize, TextRange};
 
         type Ident = String;
         \n
@@ -661,26 +666,54 @@ def write_ast_def(mod, typeinfo, f):
         textwrap.dedent(
             """
         pub struct Located<T, U = ()> {
-            pub location: Location,
-            pub end_location: Option<Location>,
+            pub range: TextRange,
             pub custom: U,
             pub node: T,
         }
     
         impl<T> Located<T> {
-            pub fn new(location: Location, end_location: Location, node: T) -> Self {
-                Self { location, end_location: Some(end_location), custom: (), node }
+            pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
+                Self { range: TextRange::new(start, end), custom: (), node }
             }
             
-            pub const fn start(&self) -> Location {
-                self.location
+            /// Creates a new node that spans the position specified by `range`.
+            pub fn with_range(node: T, range: TextRange) -> Self {
+                Self {
+                    range,
+                    custom: (),
+                    node,
+                }
+            }
+        
+            /// Returns the absolute start position of the node from the beginning of the document.
+            #[inline]
+            pub const fn start(&self) -> TextSize {
+                self.range.start()
+            }
+        
+            /// Returns the node
+            #[inline]
+            pub fn node(&self) -> &T {
+                &self.node
+            }
+        
+            /// Consumes self and returns the node.
+            #[inline]
+            pub fn into_node(self) -> T {
+                self.node
+            }
+        
+            /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
+            #[inline]
+            pub const fn range(&self) -> TextRange {
+                self.range
+            }
+        
+            /// Returns the absolute position at which the node ends in the source document.
+            #[inline]
+            pub const fn end(&self) -> TextSize {
+                self.range.end()
             }
-            
-            /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if 
-            /// [`end_location`](Located::end_location) is `None`.
-            pub fn end(&self) -> Location {
-                self.end_location.unwrap_or(self.location)
-            }      
         }
         
         impl<T, U> std::ops::Deref for Located<T, U> {

ast/src/ast_gen.rs

@@ -3,36 +3,63 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 pub use crate::constant::*;
-pub use crate::Location;
+pub use ruff_text_size::{TextRange, TextSize};
 
 type Ident = String;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Located<T, U = ()> {
-    pub location: Location,
-    pub end_location: Option<Location>,
+    pub range: TextRange,
     pub custom: U,
     pub node: T,
 }
 
 impl<T> Located<T> {
-    pub fn new(location: Location, end_location: Location, node: T) -> Self {
+    pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
         Self {
-            location,
-            end_location: Some(end_location),
+            range: TextRange::new(start, end),
             custom: (),
             node,
         }
     }
 
-    pub const fn start(&self) -> Location {
-        self.location
+    /// Creates a new node that spans the position specified by `range`.
+    pub fn with_range(node: T, range: TextRange) -> Self {
+        Self {
+            range,
+            custom: (),
+            node,
+        }
+    }
+
+    /// Returns the absolute start position of the node from the beginning of the document.
+    #[inline]
+    pub const fn start(&self) -> TextSize {
+        self.range.start()
+    }
+
+    /// Returns the node
+    #[inline]
+    pub fn node(&self) -> &T {
+        &self.node
+    }
+
+    /// Consumes self and returns the node.
+    #[inline]
+    pub fn into_node(self) -> T {
+        self.node
+    }
+
+    /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
+    #[inline]
+    pub const fn range(&self) -> TextRange {
+        self.range
     }
 
-    /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if
-    /// [`end_location`](Located::end_location) is `None`.
-    pub fn end(&self) -> Location {
-        self.end_location.unwrap_or(self.location)
+    /// Returns the absolute position at which the node ends in the source document.
+    #[inline]
+    pub const fn end(&self) -> TextSize {
+        self.range.end()
     }
 }
 
@@ -554,8 +581,7 @@ pub mod fold {
     ) -> Result<Located<MT, F::TargetU>, F::Error> {
         Ok(Located {
             custom: folder.map_user(node.custom)?,
-            location: node.location,
-            end_location: node.end_location,
+            range: node.range,
             node: f(folder, node.node)?,
         })
     }

ast/src/constant.rs

@@ -126,8 +126,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
                 Ok(crate::Expr {
                     node: expr,
                     custom: node.custom,
-                    location: node.location,
-                    end_location: node.end_location,
+                    range: node.range,
                 })
             }
             _ => crate::fold::fold_expr(self, node),
@@ -144,62 +143,55 @@ mod tests {
     fn test_constant_opt() {
         use crate::{fold::Fold, *};
 
-        let start = Default::default();
-        let end = None;
+        let range = TextRange::default();
+        #[allow(clippy::let_unit_value)]
         let custom = ();
         let ast = Located {
-            location: start,
-            end_location: end,
+            range,
             custom,
             node: ExprKind::Tuple {
                 ctx: ExprContext::Load,
                 elts: vec![
                     Located {
-                        location: start,
-                        end_location: end,
+                        range,
                         custom,
                         node: ExprKind::Constant {
                             value: BigInt::from(1).into(),
                             kind: None,
                         },
                     },
                     Located {
-                        location: start,
-                        end_location: end,
+                        range,
                         custom,
                         node: ExprKind::Constant {
                             value: BigInt::from(2).into(),
                             kind: None,
                         },
                     },
                     Located {
-                        location: start,
-                        end_location: end,
+                        range,
                         custom,
                         node: ExprKind::Tuple {
                             ctx: ExprContext::Load,
                             elts: vec![
                                 Located {
-                                    location: start,
-                                    end_location: end,
+                                    range,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(3).into(),
                                         kind: None,
                                     },
                                 },
                                 Located {
-                                    location: start,
-                                    end_location: end,
+                                    range,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(4).into(),
                                         kind: None,
                                     },
                                 },
                                 Located {
-                                    location: start,
-                                    end_location: end,
+                                    range,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(5).into(),
@@ -218,8 +210,7 @@ mod tests {
         assert_eq!(
             new_ast,
             Located {
-                location: start,
-                end_location: end,
+                range,
                 custom,
                 node: ExprKind::Constant {
                     value: Constant::Tuple(vec![

ast/src/lib.rs

@@ -7,6 +7,5 @@ mod impls;
 mod unparse;
 
 pub use ast_gen::*;
-pub use rustpython_compiler_core::Location;
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;

core/Cargo.toml

@@ -9,10 +9,11 @@ license = "MIT"
 
 [dependencies]
 bitflags = { workspace = true }
-bstr = { workspace = true }
 itertools = { workspace = true }
 num-bigint = { workspace = true }
 num-complex = { workspace = true }
 serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
+ruff_text_size =  { path = "../ruff_text_size" }
 
 lz4_flex = "0.9.2"
+

core/src/error.rs

@@ -1,11 +1,11 @@
-use crate::Location;
+use ruff_text_size::TextSize;
 use std::error::Error as StdError;
 use std::fmt::Display;
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct BaseError<T> {
     pub error: T,
-    pub location: Location,
+    pub location: TextSize,
     pub source_path: String,
 }
 
@@ -31,7 +31,12 @@ where
     T: std::fmt::Display,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        self.location.fmt_with(f, &self.error)
+        write!(
+            f,
+            "{} at byte offset {}",
+            &self.error,
+            u32::from(self.location)
+        )
     }
 }
 

parser/Cargo.toml

@@ -21,6 +21,7 @@ tiny-keccak = { version = "2", features = ["sha3"] }
 [dependencies]
 rustpython-ast = { path = "../ast", version = "0.2.0" }
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
+ruff_text_size =  { path = "../ruff_text_size" }
 
 ahash = { workspace = true }
 itertools = { workspace = true }

parser/src/function.rs

@@ -4,6 +4,7 @@ use crate::{
     ast,
     lexer::{LexicalError, LexicalErrorType},
 };
+use ruff_text_size::TextSize;
 use rustc_hash::FxHashSet;
 
 pub(crate) struct ArgumentList {
@@ -83,10 +84,7 @@ pub(crate) fn parse_params(
     Ok((pos_only, names, defaults))
 }
 
-type FunctionArgument = (
-    Option<(ast::Location, ast::Location, Option<String>)>,
-    ast::Expr,
-);
+type FunctionArgument = (Option<(TextSize, TextSize, Option<String>)>, ast::Expr);
 
 // Parse arguments as supplied during a function/lambda *call*.
 pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {