Skip to content
← Back to Examples

URL Parser

RustRubyurls networkbeginner

Parse URLs (Uniform Resource Locators) into structured components including scheme, host, port, path, query parameters, and fragment.

Key Concepts

  • scheme detection
  • host parsing
  • query string parsing
  • component extraction
  • optional parts

Why Parse This Format?

URLs are fundamental to web development, API design, and routing systems. Parsing URLs correctly is essential for web servers, clients, routers, and security validation.

Use Cases

  • Web server request routing
  • API endpoint parsing
  • Web crawler link extraction
  • Security validation of user-submitted URLs
  • Link normalization and canonicalization

Input Format

URLs follow a standard format: scheme://host[:port][/path][?query][#fragment]. The scheme is required, as is the host. Port, path, query, and fragment are optional.

scheme://host[:port][/path][?query][#fragment]

Examples

https://example.comVALID

Simple HTTPS URL

http://localhost:8080/api/usersVALID

URL with port and path

https://github.com/user/repo?tab=issuesVALID

URL with query parameters

ftp://ftp.example.com:21/files/doc.pdf#page=5VALID

Full URL with all components

https://search.example.com:443/search?q=rust+parser&page=1#resultsVALID

Complex URL with query and fragment

example.comINVALID

Invalid: missing scheme

://example.comINVALID

Invalid: missing scheme name

Output Format

The parser returns a ParsedUrl struct with each URL component as a separate field, with proper types for optional components.

Structure

scheme

String - the protocol (http, https, ftp, etc.)

host

String - the domain or IP address

port

Option<u16> - optional port number

path

Option<String> - optional path component

query

Option<HashMap<String, String>> - parsed query parameters

fragment

Option<String> - optional fragment/anchor

Visual Overview

URL Parser diagram

How the Code Works

This section explains how the parsing rules map to the format semantics.

Scheme Detection

.rule("scheme", choice(vec![
    dynamic(str("https")),
    dynamic(str("http")),
    dynamic(str("ftp")),
    dynamic(re("[a-z]+")),
]))
.rule("scheme_sep", str("://"))

The parser first identifies the URL scheme (protocol): Schemes are case-insensitive and followed by `://`. Common schemes are recognized explicitly, with a fallback regex for custom schemes.

Host and Port Parsing

.rule("host", re(r#"[a-zA-Z0-9.-]+"#))
.rule("port", seq(vec![dynamic(str(":")), dynamic(re("[0-9]+"))]))

The host can be a domain name, IP address, or localhost: Port is optional and defaults based on scheme (80 for http, 443 for https). The parser extracts it when present.

Path Component

.rule("path", seq(vec![
    dynamic(str("/")),
    dynamic(re(r#"[^?#]*"#)),
]))

The path starts with `/` and continues until `?`, `#`, or end: Paths may contain multiple segments separated by `/`, and each segment can have URL-encoded characters.

Query String Parsing

.rule("query", seq(vec![
    dynamic(str("?")),
    dynamic(re(r#"[^#]*"#)),
]))

Query strings are parsed into key-value pairs: The query string is further parsed into a HashMap, handling `&` separators and `=` assignments.

Fragment Extraction

.rule("fragment", seq(vec![
    dynamic(str("#")),
    dynamic(re(r#".*"#)),
]))

The fragment (anchor) is the final component: Fragments are used for in-page navigation and are not sent to the server.

Fragment Extraction

.rule("fragment", seq(vec![
    dynamic(str("#")),
    dynamic(re(r#".*"#)),
]))

The fragment (anchor) is the final component: Fragments are used for in-page navigation and are not sent to the server.

Source Code

Rust
//! URL Parser Example
//!
//! This example demonstrates parsing URLs into structured components.
//! Shows how to use sequence, choice, and repetition combinators.
//!
//! Run with: cargo run --example url_parser --no-default-features

use parsanol::portable::{
    parser_dsl::{dynamic, re, seq, str, GrammarBuilder, ParsletExt},
    AstArena, Grammar, PortableParser,
};
use std::collections::HashMap;

/// Build a URL grammar
fn build_url_grammar() -> Grammar {
    GrammarBuilder::new()
        // Scheme: http, https, ftp, etc.
        .rule("scheme", re("[a-z][a-z0-9+.-]*"))
        // Host: domain or IP
        .rule("host", re("[a-zA-Z0-9][a-zA-Z0-9.-]*"))
        // Port: optional :8080
        .rule("port", re(":[0-9]+"))
        // Path: /foo/bar
        .rule("path_segment", re("[^/?#]+"))
        .rule("path", re("/[^?#]*"))
        // Query: ?key=value&...
        .rule("query_string", re("\\?[^#]*"))
        // Fragment: #anchor
        .rule("fragment", re("#.*"))
        // Full URL
        .rule(
            "url",
            seq(vec![
                dynamic(re("[a-z][a-z0-9+.-]*")), // scheme
                dynamic(str("://")),
                dynamic(re("[a-zA-Z0-9][a-zA-Z0-9.-]*")), // host
                dynamic(re(":[0-9]+").optional()),        // optional port
                dynamic(re("/[^?#]*").optional()),        // optional path
                dynamic(re("\\?[^#]*").optional()),       // optional query
                dynamic(re("#.*").optional()),            // optional fragment
            ]),
        )
        .build()
}

/// Parsed URL components
#[derive(Debug, Clone)]
pub struct ParsedUrl {
    pub scheme: String,
    pub host: String,
    pub port: Option<u16>,
    pub path: Option<String>,
    pub query: Option<HashMap<String, String>>,
    pub fragment: Option<String>,
}

/// Parse a URL string into structured components
pub fn parse_url(input: &str) -> Result<ParsedUrl, String> {
    let grammar = build_url_grammar();
    let mut arena = AstArena::for_input(input.len());
    let mut parser = PortableParser::new(&grammar, input, &mut arena);
    let _ast = parser
        .parse()
        .map_err(|e| format!("Parse error: {:?}", e))?;

    // Manual parsing for simplicity (the grammar validates structure)
    let input = input.trim();

    // Split scheme
    let (scheme, rest) = input.split_once("://").ok_or("Missing :// in URL")?;

    // Split fragment
    let (rest, fragment) = if let Some(idx) = rest.find('#') {
        (&rest[..idx], Some(rest[idx + 1..].to_string()))
    } else {
        (rest, None)
    };

    // Split query
    let (rest, query) = if let Some(idx) = rest.find('?') {
        let query_str = &rest[idx + 1..];
        let query = parse_query_string(query_str);
        (&rest[..idx], Some(query))
    } else {
        (rest, None)
    };

    // Split path
    let (host_port, path) = if let Some(idx) = rest.find('/') {
        (&rest[..idx], Some(rest[idx..].to_string()))
    } else {
        (rest, None)
    };

    // Split port
    let (host, port) = if let Some(idx) = host_port.rfind(':') {
        let port_str = &host_port[idx + 1..];
        let port = port_str.parse::<u16>().ok();
        (&host_port[..idx], port)
    } else {
        (host_port, None)
    };

    Ok(ParsedUrl {
        scheme: scheme.to_string(),
        host: host.to_string(),
        port,
        path,
        query,
        fragment,
    })
}

/// Parse query string into key-value pairs
fn parse_query_string(query: &str) -> HashMap<String, String> {
    let mut map = HashMap::new();
    for pair in query.split('&') {
        if let Some((key, value)) = pair.split_once('=') {
            map.insert(key.to_string(), value.to_string());
        } else if !pair.is_empty() {
            map.insert(pair.to_string(), String::new());
        }
    }
    map
}

fn main() {
    println!("URL Parser Example");
    println!("==================\n");

    let urls = [
        "https://example.com",
        "http://localhost:8080/api",
        "https://github.com/user/repo?tab=issues",
        "ftp://ftp.example.com:21/files/document.pdf#page=5",
        "https://search.example.com:443/search?q=rust+parser&page=1#results",
    ];

    for url in urls {
        println!("Input: {}", url);
        match parse_url(url) {
            Ok(parsed) => {
                println!("  scheme:   {}", parsed.scheme);
                println!("  host:     {}", parsed.host);
                println!("  port:     {:?}", parsed.port);
                println!("  path:     {:?}", parsed.path);
                println!("  query:    {:?}", parsed.query);
                println!("  fragment: {:?}", parsed.fragment);
            }
            Err(e) => println!("  Error: {}", e),
        }
        println!();
    }
}
Ruby
# frozen_string_literal: true

# URL Parser Example - Basic Parsing
#
# This example demonstrates parsing URLs into their components.
# Shows protocol, host, port, path, query string, and fragment parsing.
#
# Run with: ruby -Ilib example/url_ruby_transform.rb

$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"

require "parsanol"

# Step 1: Define the URL grammar
class UrlParser < Parsanol::Parser
  root :url

  rule(:url) do
    protocol.as(:protocol) >>
      str("://") >>
      host.as(:host) >>
      port.maybe.as(:port) >>
      path.maybe.as(:path) >>
      query.maybe.as(:query) >>
      fragment.maybe.as(:fragment)
  end

  rule(:protocol) do
    str("http") | str("https") | str("ftp") | str("ws") | str("wss")
  end

  rule(:host) do
    (domain | ip_address).as(:address)
  end

  rule(:domain) do
    label >> (str(".") >> label).repeat
  end

  rule(:label) do
    match("[a-zA-Z0-9]").repeat(1)
  end

  rule(:ip_address) do
    octet >> str(".") >> octet >> str(".") >> octet >> str(".") >> octet
  end

  rule(:octet) do
    match("[0-9]").repeat(1, 3)
  end

  rule(:port) do
    str(":") >> match("[0-9]").repeat(1).as(:number)
  end

  rule(:path) do
    str("/") >> path_segment.repeat(1).as(:segments)
  end

  rule(:path_segment) do
    match("[^/?#]").repeat(1) >> str("/").maybe
  end

  rule(:query) do
    str("?") >> query_string.as(:string)
  end

  rule(:query_string) do
    match("[^#]").repeat
  end

  rule(:fragment) do
    str("#") >> match(".").repeat.as(:value)
  end
end

# Step 2: URL class
class ParsedURL
  attr_reader :protocol, :host, :port, :path, :query, :fragment

  def initialize(protocol:, host:, port: nil, path: nil, query: nil,
fragment: nil)
    @protocol = protocol
    @host = host
    @port = port
    @path = path
    @query = query
    @fragment = fragment
  end

  def to_s
    url = "#{@protocol}://#{@host}"
    url += ":#{@port}" if @port
    url += @path if @path
    url += "?#{@query}" if @query
    url += "##{@fragment}" if @fragment
    url
  end

  def path_segments
    @path ? @path.split("/").reject(&:empty?) : []
  end

  def query_params
    return {} unless @query

    @query.split("&").each_with_object({}) do |pair, hash|
      key, value = pair.split("=", 2)
      hash[key] = value || ""
    end
  end
end

# Step 3: Transform
class UrlTransform < Parsanol::Transform
  rule(
    protocol: simple(:p),
    host: simple(:h),
    port: simple(:port),
    path: simple(:path),
    query: simple(:q),
    fragment: simple(:f),
  ) do
    port_num = port&.dig(:number)&.to_i
    ParsedURL.new(
      protocol: p.to_s,
      host: h[:address].to_s,
      port: port_num,
      path: path&.dig(:segments)&.to_s,
      query: q&.dig(:string)&.to_s,
      fragment: f.to_s,
    )
  end

  rule(
    protocol: simple(:p),
    host: simple(:h),
    port: simple(:port),
    path: simple(:path),
    query: simple(:q),
  ) do
    port_num = port&.dig(:number)&.to_i
    ParsedURL.new(
      protocol: p.to_s,
      host: h[:address].to_s,
      port: port_num,
      path: path&.dig(:segments)&.to_s,
      query: q&.dig(:string)&.to_s,
    )
  end

  rule(
    protocol: simple(:p),
    host: simple(:h),
    port: simple(:port),
    path: simple(:path),
  ) do
    port_num = port&.dig(:number)&.to_i
    ParsedURL.new(
      protocol: p.to_s,
      host: h[:address].to_s,
      port: port_num,
      path: path&.dig(:segments)&.to_s,
    )
  end

  rule(protocol: simple(:p), host: simple(:h), port: simple(:port)) do
    ParsedURL.new(
      protocol: p.to_s,
      host: h[:address].to_s,
      port: port&.dig(:number)&.to_i,
    )
  end

  rule(protocol: simple(:p), host: simple(:h)) do
    ParsedURL.new(protocol: p.to_s, host: h[:address].to_s)
  end
end

def parse_url(input)
  parser = UrlParser.new
  tree = parser.parse(input)

  transform = UrlTransform.new
  transform.apply(tree)
rescue Parsanol::ParseFailed => e
  puts "Parse failed: #{e.message}"
  nil
end

if __FILE__ == $PROGRAM_NAME
  puts "=" * 60
  puts "URL Parser - Basic Parsing"
  puts "=" * 60
  puts

  test_urls = [
    "http://example.com",
    "https://example.com:8080",
    "https://example.com/path/to/resource",
    "https://example.com/search?q=ruby&limit=10",
    "https://example.com/page#section",
    "https://api.example.com:443/v1/users?id=123#results",
    "http://192.168.1.1:3000/admin",
    "ws://websocket.example.com/socket",
  ]

  test_urls.each do |url_str|
    puts "-" * 40
    puts "Input: #{url_str}"
    url = parse_url(url_str)
    if url
      puts "  Protocol: #{url.protocol}"
      puts "  Host: #{url.host}"
      puts "  Port: #{url.port || '(default)'}"
      puts "  Path: #{url.path || '/'}"
      puts "  Query: #{url.query || '(none)'}"
      puts "  Fragment: #{url.fragment || '(none)'}"
      puts "  Reconstructed: #{url}"
    end
    puts
  end
end