Comments Parser

RustRubytext processingbeginner

Parse source code with line comments (//) and block comments (/* */), demonstrating how to handle nested structures and comment stripping.

Key Concepts

line comments
block comments
comment stripping
code element extraction
multi-line handling

Why Parse This Format?

Comment handling is essential for source code processing, documentation generation, and code analysis tools. Stripping comments is useful for minification and preprocessing.

Use Cases

Source code preprocessing
Documentation extraction
Code minification
Static analysis tools
Syntax highlighting

Input Format

Source code containing line comments (// to end of line) and block comments (/* to */). Block comments can span multiple lines.

line_comment = // text\n | block_comment = /* text */

Examples

// This is a line comment\nvariableVALID

Line comment before code

variable // inline commentVALID

Inline line comment

/* This is a\n block comment */VALID

Multi-line block comment

var1 /* inline block */ var2VALID

Inline block comment

// comment\nvar1\n/* block\ncomment */\nvar2VALID

Mixed comment types

/* unclosed commentINVALID

Invalid: missing closing */

Output Format

The parser returns a list of CodeElement enum values distinguishing between identifiers, comments, and whitespace.

Structure

CodeElement

Enum: Identifier(String), Comment(Comment), Newline

Comment

Enum: Line(String), Block(String)

Visual Overview

How the Code Works

This section explains how the parsing rules map to the format semantics.

Line Comment Parsing

.rule("line_comment", seq(vec![
    dynamic(str("//")),
    dynamic(re("[^\n\r]*")),
]))

Line comments start with `//` and continue to end of line: The regex `[^\n\r]*` matches any character except newlines.

Block Comment Parsing

.rule("block_comment", seq(vec![
    dynamic(str("/*")),
    dynamic(re("(?:[^*]|\\*(?![/))*")),  // Not */ or * followed by /
    dynamic(str("*/")),
]))

Block comments use `/* */` delimiters: The pattern `(?:[^*]|\*(?![/])*` matches anything that isn't `*/` using negative lookahead.

Comment Extraction

// Line comment
if pos + 1 < chars.len() && chars[pos] == '/' && chars[pos + 1] == '/' {
    pos += 2;
    let start = pos;
    while pos < chars.len() && chars[pos] != '\n' && chars[pos] != '\r' {
        pos += 1;
    }
    let comment: String = chars[start..pos].iter().collect();
    elements.push(CodeElement::Comment(Comment::Line(comment)));
}

// Block comment
if pos + 1 < chars.len() && chars[pos] == '/' && chars[pos + 1] == '*' {
    pos += 2;
    let start = pos;
    while pos + 1 < chars.len() && !(chars[pos] == '*' && chars[pos + 1] == '/') {
        pos += 1;
    }
    // ...
}

Comments are extracted by scanning character by character:

Comment Stripping

pub fn strip_comments(input: &str) -> String {
    let elements = parse_with_comments(input).unwrap_or_default();
    let mut result = String::new();

    for element in elements {
        match element {
            CodeElement::Identifier(ident) => result.push_str(&ident),
            CodeElement::Newline => result.push('\n'),
            CodeElement::Comment(_) => { /* Skip comments */ }
        }
    }
    result.trim().to_string()
}

Comments can be removed from code:

Comment Stripping

pub fn strip_comments(input: &str) -> String {
    let elements = parse_with_comments(input).unwrap_or_default();
    let mut result = String::new();

    for element in elements {
        match element {
            CodeElement::Identifier(ident) => result.push_str(&ident),
            CodeElement::Newline => result.push('\n'),
            CodeElement::Comment(_) => { /* Skip comments */ }
        }
    }
    result.trim().to_string()
}

Comments can be removed from code:

Source Code

Rust

//! Comment Parser Example
//!
//! This example demonstrates parsing code with line and block comments.
//! Shows handling of nested structures with comment skipping.
//! Based on the Parslet comments.rb example.
//!
//! Run with: cargo run --example comments_parser --no-default-features

#![allow(clippy::print_literal)]

use parsanol::portable::{
    parser_dsl::{choice, dynamic, re, seq, str, GrammarBuilder},
    AstArena, Grammar, PortableParser,
};

/// Build a grammar that handles comments
fn build_comment_grammar() -> Grammar {
    GrammarBuilder::new()
        // Line comment: // until end of line
        .rule("line_comment", seq(vec![
            dynamic(str("//")),
            dynamic(re("[^\n\r]*")),
        ]))
        // Block comment: /* ... */
        .rule("block_comment", seq(vec![
            dynamic(str("/*")),
            dynamic(re("(?:[^*]|\\*(?![/]))*")),  // Not */ or * followed by /
            dynamic(str("*/")),
        ]))
        // Whitespace
        .rule("space", re("[ \\t]+"))
        // Identifier
        .rule("identifier", re("[a-zA-Z_][a-zA-Z0-9_]*"))
        // Statement: identifier or comment
        .rule("statement", choice(vec![
            dynamic(re("[a-zA-Z_][a-zA-Z0-9_]*")),
            dynamic(seq(vec![
                dynamic(str("//")),
                dynamic(re("[^\n\r]*")),
            ])),
            dynamic(seq(vec![
                dynamic(str("/*")),
                dynamic(re("(?:[^*]|\\*(?![/]))*")),
                dynamic(str("*/")),
            ])),
        ]))
        // Line: optional statement
        .rule("line", re("[ \\t]*(?:(?:[a-zA-Z_][a-zA-Z0-9_]*)|(?://[^\n\r]*)|(?:/\\*(?:[^*]|\\*(?![/]))*\\*/))?[ \\t]*"))
        // Document: multiple lines
        .rule("document", re(".*"))
        .build()
}

/// Comment types
#[derive(Debug, Clone)]
pub enum Comment {
    Line(String),
    Block(String),
}

/// Parsed code element
#[derive(Debug, Clone)]
pub enum CodeElement {
    Identifier(String),
    Comment(Comment),
    Newline,
}

/// Parse code with comments
pub fn parse_with_comments(input: &str) -> Result<Vec<CodeElement>, String> {
    let grammar = build_comment_grammar();
    let mut arena = AstArena::for_input(input.len());
    let mut parser = PortableParser::new(&grammar, input, &mut arena);

    parser
        .parse()
        .map_err(|e| format!("Parse error: {:?}", e))?;

    // Manual parsing for element extraction
    let mut elements = Vec::new();
    let mut pos = 0;
    let chars: Vec<char> = input.chars().collect();

    while pos < chars.len() {
        // Skip whitespace (except newlines)
        while pos < chars.len() && (chars[pos] == ' ' || chars[pos] == '\t') {
            pos += 1;
        }

        if pos >= chars.len() {
            break;
        }

        // Newline
        if chars[pos] == '\n' || chars[pos] == '\r' {
            elements.push(CodeElement::Newline);
            pos += 1;
            if chars[pos - 1] == '\r' && pos < chars.len() && chars[pos] == '\n' {
                pos += 1;
            }
            continue;
        }

        // Line comment
        if pos + 1 < chars.len() && chars[pos] == '/' && chars[pos + 1] == '/' {
            pos += 2;
            let start = pos;
            while pos < chars.len() && chars[pos] != '\n' && chars[pos] != '\r' {
                pos += 1;
            }
            let comment: String = chars[start..pos].iter().collect();
            elements.push(CodeElement::Comment(Comment::Line(comment)));
            continue;
        }

        // Block comment
        if pos + 1 < chars.len() && chars[pos] == '/' && chars[pos + 1] == '*' {
            pos += 2;
            let start = pos;
            while pos + 1 < chars.len() && !(chars[pos] == '*' && chars[pos + 1] == '/') {
                pos += 1;
            }
            let comment: String = chars[start..pos].iter().collect();
            pos += 2; // Skip */
            elements.push(CodeElement::Comment(Comment::Block(comment)));
            continue;
        }

        // Identifier
        if chars[pos].is_alphabetic() || chars[pos] == '_' {
            let start = pos;
            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
                pos += 1;
            }
            let ident: String = chars[start..pos].iter().collect();
            elements.push(CodeElement::Identifier(ident));
            continue;
        }

        // Skip unknown character
        pos += 1;
    }

    Ok(elements)
}

/// Strip comments from code
pub fn strip_comments(input: &str) -> String {
    let elements = parse_with_comments(input).unwrap_or_default();
    let mut result = String::new();

    for element in elements {
        match element {
            CodeElement::Identifier(ident) => {
                if !result.is_empty() && !result.ends_with('\n') && !result.ends_with(' ') {
                    result.push(' ');
                }
                result.push_str(&ident);
            }
            CodeElement::Newline => {
                result.push('\n');
            }
            CodeElement::Comment(_) => {
                // Skip comments
            }
        }
    }

    result.trim().to_string()
}

fn main() {
    println!("Comment Parser Example");
    println!("======================\n");

    let code = r#"
// This is a line comment
variable1
variable2 // inline comment
/* This is a
   block comment */
variable3 /* inline block */ variable4
"#;

    println!("Input Code:");
    println!("-----------");
    println!("{}", code);

    println!("\nParsed Elements:");
    println!("----------------");
    match parse_with_comments(code) {
        Ok(elements) => {
            for element in elements {
                match element {
                    CodeElement::Identifier(ident) => println!("  IDENT: {}", ident),
                    CodeElement::Comment(Comment::Line(c)) => {
                        println!("  LINE_COMMENT: {}", c.trim())
                    }
                    CodeElement::Comment(Comment::Block(c)) => {
                        println!("  BLOCK_COMMENT: {}", c.trim().replace('\n', "\\n"))
                    }
                    CodeElement::Newline => println!("  NEWLINE"),
                }
            }
        }
        Err(e) => println!("Error: {}", e),
    }

    println!("\nStripped Code (no comments):");
    println!("----------------------------");
    println!("{}", strip_comments(code));
}

Ruby

# frozen_string_literal: true

# A small example on how to parse common types of comments. The example
# started out with parser code from Stephen Waits.

$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"

require "pp"
require "parsanol/parslet"
require "parsanol/convenience"

class ALanguage < Parsanol::Parser
  root(:lines)

  rule(:lines) { line.repeat }
  rule(:line) { spaces >> expression.repeat >> newline }
  rule(:newline) { str("\n") >> str("\r").maybe }

  rule(:expression) { (str("a").as(:a) >> spaces).as(:exp) }

  rule(:spaces) { space.repeat }
  rule(:space) { multiline_comment | line_comment | str(" ") }

  rule(:line_comment) do
    (str("//") >> (newline.absent? >> any).repeat).as(:line)
  end
  rule(:multiline_comment) do
    (str("/*") >> (str("*/").absent? >> any).repeat >> str("*/")).as(:multi)
  end
end

code = '
  a
  // line comment
  a a a // line comment
  a /* inline comment */ a
  /* multiline
  comment */
'

pp ALanguage.new.parse_with_debug(code)