URL Parser
Parse URLs (Uniform Resource Locators) into structured components including scheme, host, port, path, query parameters, and fragment.
Key Concepts
- scheme detection
- host parsing
- query string parsing
- component extraction
- optional parts
Why Parse This Format?
URLs are fundamental to web development, API design, and routing systems. Parsing URLs correctly is essential for web servers, clients, routers, and security validation.
Use Cases
- Web server request routing
- API endpoint parsing
- Web crawler link extraction
- Security validation of user-submitted URLs
- Link normalization and canonicalization
Input Format
URLs follow a standard format: scheme://host[:port][/path][?query][#fragment]. The scheme is required, as is the host. Port, path, query, and fragment are optional.
scheme://host[:port][/path][?query][#fragment]Examples
https://example.comVALIDSimple HTTPS URL
http://localhost:8080/api/usersVALIDURL with port and path
https://github.com/user/repo?tab=issuesVALIDURL with query parameters
ftp://ftp.example.com:21/files/doc.pdf#page=5VALIDFull URL with all components
https://search.example.com:443/search?q=rust+parser&page=1#resultsVALIDComplex URL with query and fragment
example.comINVALIDInvalid: missing scheme
://example.comINVALIDInvalid: missing scheme name
Output Format
The parser returns a ParsedUrl struct with each URL component as a separate field, with proper types for optional components.
Structure
schemeString - the protocol (http, https, ftp, etc.)
hostString - the domain or IP address
portOption<u16> - optional port number
pathOption<String> - optional path component
queryOption<HashMap<String, String>> - parsed query parameters
fragmentOption<String> - optional fragment/anchor
Visual Overview
How the Code Works
This section explains how the parsing rules map to the format semantics.
Scheme Detection
.rule("scheme", choice(vec![
dynamic(str("https")),
dynamic(str("http")),
dynamic(str("ftp")),
dynamic(re("[a-z]+")),
]))
.rule("scheme_sep", str("://"))The parser first identifies the URL scheme (protocol): Schemes are case-insensitive and followed by `://`. Common schemes are recognized explicitly, with a fallback regex for custom schemes.
Host and Port Parsing
.rule("host", re(r#"[a-zA-Z0-9.-]+"#))
.rule("port", seq(vec![dynamic(str(":")), dynamic(re("[0-9]+"))]))The host can be a domain name, IP address, or localhost: Port is optional and defaults based on scheme (80 for http, 443 for https). The parser extracts it when present.
Path Component
.rule("path", seq(vec![
dynamic(str("/")),
dynamic(re(r#"[^?#]*"#)),
]))The path starts with `/` and continues until `?`, `#`, or end: Paths may contain multiple segments separated by `/`, and each segment can have URL-encoded characters.
Query String Parsing
.rule("query", seq(vec![
dynamic(str("?")),
dynamic(re(r#"[^#]*"#)),
]))Query strings are parsed into key-value pairs: The query string is further parsed into a HashMap, handling `&` separators and `=` assignments.
Fragment Extraction
.rule("fragment", seq(vec![
dynamic(str("#")),
dynamic(re(r#".*"#)),
]))The fragment (anchor) is the final component: Fragments are used for in-page navigation and are not sent to the server.
Fragment Extraction
.rule("fragment", seq(vec![
dynamic(str("#")),
dynamic(re(r#".*"#)),
]))The fragment (anchor) is the final component: Fragments are used for in-page navigation and are not sent to the server.
Source Code
//! URL Parser Example
//!
//! This example demonstrates parsing URLs into structured components.
//! Shows how to use sequence, choice, and repetition combinators.
//!
//! Run with: cargo run --example url_parser --no-default-features
use parsanol::portable::{
parser_dsl::{dynamic, re, seq, str, GrammarBuilder, ParsletExt},
AstArena, Grammar, PortableParser,
};
use std::collections::HashMap;
/// Build a URL grammar
fn build_url_grammar() -> Grammar {
GrammarBuilder::new()
// Scheme: http, https, ftp, etc.
.rule("scheme", re("[a-z][a-z0-9+.-]*"))
// Host: domain or IP
.rule("host", re("[a-zA-Z0-9][a-zA-Z0-9.-]*"))
// Port: optional :8080
.rule("port", re(":[0-9]+"))
// Path: /foo/bar
.rule("path_segment", re("[^/?#]+"))
.rule("path", re("/[^?#]*"))
// Query: ?key=value&...
.rule("query_string", re("\\?[^#]*"))
// Fragment: #anchor
.rule("fragment", re("#.*"))
// Full URL
.rule(
"url",
seq(vec![
dynamic(re("[a-z][a-z0-9+.-]*")), // scheme
dynamic(str("://")),
dynamic(re("[a-zA-Z0-9][a-zA-Z0-9.-]*")), // host
dynamic(re(":[0-9]+").optional()), // optional port
dynamic(re("/[^?#]*").optional()), // optional path
dynamic(re("\\?[^#]*").optional()), // optional query
dynamic(re("#.*").optional()), // optional fragment
]),
)
.build()
}
/// Parsed URL components
#[derive(Debug, Clone)]
pub struct ParsedUrl {
pub scheme: String,
pub host: String,
pub port: Option<u16>,
pub path: Option<String>,
pub query: Option<HashMap<String, String>>,
pub fragment: Option<String>,
}
/// Parse a URL string into structured components
pub fn parse_url(input: &str) -> Result<ParsedUrl, String> {
let grammar = build_url_grammar();
let mut arena = AstArena::for_input(input.len());
let mut parser = PortableParser::new(&grammar, input, &mut arena);
let _ast = parser
.parse()
.map_err(|e| format!("Parse error: {:?}", e))?;
// Manual parsing for simplicity (the grammar validates structure)
let input = input.trim();
// Split scheme
let (scheme, rest) = input.split_once("://").ok_or("Missing :// in URL")?;
// Split fragment
let (rest, fragment) = if let Some(idx) = rest.find('#') {
(&rest[..idx], Some(rest[idx + 1..].to_string()))
} else {
(rest, None)
};
// Split query
let (rest, query) = if let Some(idx) = rest.find('?') {
let query_str = &rest[idx + 1..];
let query = parse_query_string(query_str);
(&rest[..idx], Some(query))
} else {
(rest, None)
};
// Split path
let (host_port, path) = if let Some(idx) = rest.find('/') {
(&rest[..idx], Some(rest[idx..].to_string()))
} else {
(rest, None)
};
// Split port
let (host, port) = if let Some(idx) = host_port.rfind(':') {
let port_str = &host_port[idx + 1..];
let port = port_str.parse::<u16>().ok();
(&host_port[..idx], port)
} else {
(host_port, None)
};
Ok(ParsedUrl {
scheme: scheme.to_string(),
host: host.to_string(),
port,
path,
query,
fragment,
})
}
/// Parse query string into key-value pairs
fn parse_query_string(query: &str) -> HashMap<String, String> {
let mut map = HashMap::new();
for pair in query.split('&') {
if let Some((key, value)) = pair.split_once('=') {
map.insert(key.to_string(), value.to_string());
} else if !pair.is_empty() {
map.insert(pair.to_string(), String::new());
}
}
map
}
fn main() {
println!("URL Parser Example");
println!("==================\n");
let urls = [
"https://example.com",
"http://localhost:8080/api",
"https://github.com/user/repo?tab=issues",
"ftp://ftp.example.com:21/files/document.pdf#page=5",
"https://search.example.com:443/search?q=rust+parser&page=1#results",
];
for url in urls {
println!("Input: {}", url);
match parse_url(url) {
Ok(parsed) => {
println!(" scheme: {}", parsed.scheme);
println!(" host: {}", parsed.host);
println!(" port: {:?}", parsed.port);
println!(" path: {:?}", parsed.path);
println!(" query: {:?}", parsed.query);
println!(" fragment: {:?}", parsed.fragment);
}
Err(e) => println!(" Error: {}", e),
}
println!();
}
}
# frozen_string_literal: true
# URL Parser Example - Basic Parsing
#
# This example demonstrates parsing URLs into their components.
# Shows protocol, host, port, path, query string, and fragment parsing.
#
# Run with: ruby -Ilib example/url_ruby_transform.rb
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
require "parsanol"
# Step 1: Define the URL grammar
class UrlParser < Parsanol::Parser
root :url
rule(:url) do
protocol.as(:protocol) >>
str("://") >>
host.as(:host) >>
port.maybe.as(:port) >>
path.maybe.as(:path) >>
query.maybe.as(:query) >>
fragment.maybe.as(:fragment)
end
rule(:protocol) do
str("http") | str("https") | str("ftp") | str("ws") | str("wss")
end
rule(:host) do
(domain | ip_address).as(:address)
end
rule(:domain) do
label >> (str(".") >> label).repeat
end
rule(:label) do
match("[a-zA-Z0-9]").repeat(1)
end
rule(:ip_address) do
octet >> str(".") >> octet >> str(".") >> octet >> str(".") >> octet
end
rule(:octet) do
match("[0-9]").repeat(1, 3)
end
rule(:port) do
str(":") >> match("[0-9]").repeat(1).as(:number)
end
rule(:path) do
str("/") >> path_segment.repeat(1).as(:segments)
end
rule(:path_segment) do
match("[^/?#]").repeat(1) >> str("/").maybe
end
rule(:query) do
str("?") >> query_string.as(:string)
end
rule(:query_string) do
match("[^#]").repeat
end
rule(:fragment) do
str("#") >> match(".").repeat.as(:value)
end
end
# Step 2: URL class
class ParsedURL
attr_reader :protocol, :host, :port, :path, :query, :fragment
def initialize(protocol:, host:, port: nil, path: nil, query: nil,
fragment: nil)
@protocol = protocol
@host = host
@port = port
@path = path
@query = query
@fragment = fragment
end
def to_s
url = "#{@protocol}://#{@host}"
url += ":#{@port}" if @port
url += @path if @path
url += "?#{@query}" if @query
url += "##{@fragment}" if @fragment
url
end
def path_segments
@path ? @path.split("/").reject(&:empty?) : []
end
def query_params
return {} unless @query
@query.split("&").each_with_object({}) do |pair, hash|
key, value = pair.split("=", 2)
hash[key] = value || ""
end
end
end
# Step 3: Transform
class UrlTransform < Parsanol::Transform
rule(
protocol: simple(:p),
host: simple(:h),
port: simple(:port),
path: simple(:path),
query: simple(:q),
fragment: simple(:f),
) do
port_num = port&.dig(:number)&.to_i
ParsedURL.new(
protocol: p.to_s,
host: h[:address].to_s,
port: port_num,
path: path&.dig(:segments)&.to_s,
query: q&.dig(:string)&.to_s,
fragment: f.to_s,
)
end
rule(
protocol: simple(:p),
host: simple(:h),
port: simple(:port),
path: simple(:path),
query: simple(:q),
) do
port_num = port&.dig(:number)&.to_i
ParsedURL.new(
protocol: p.to_s,
host: h[:address].to_s,
port: port_num,
path: path&.dig(:segments)&.to_s,
query: q&.dig(:string)&.to_s,
)
end
rule(
protocol: simple(:p),
host: simple(:h),
port: simple(:port),
path: simple(:path),
) do
port_num = port&.dig(:number)&.to_i
ParsedURL.new(
protocol: p.to_s,
host: h[:address].to_s,
port: port_num,
path: path&.dig(:segments)&.to_s,
)
end
rule(protocol: simple(:p), host: simple(:h), port: simple(:port)) do
ParsedURL.new(
protocol: p.to_s,
host: h[:address].to_s,
port: port&.dig(:number)&.to_i,
)
end
rule(protocol: simple(:p), host: simple(:h)) do
ParsedURL.new(protocol: p.to_s, host: h[:address].to_s)
end
end
def parse_url(input)
parser = UrlParser.new
tree = parser.parse(input)
transform = UrlTransform.new
transform.apply(tree)
rescue Parsanol::ParseFailed => e
puts "Parse failed: #{e.message}"
nil
end
if __FILE__ == $PROGRAM_NAME
puts "=" * 60
puts "URL Parser - Basic Parsing"
puts "=" * 60
puts
test_urls = [
"http://example.com",
"https://example.com:8080",
"https://example.com/path/to/resource",
"https://example.com/search?q=ruby&limit=10",
"https://example.com/page#section",
"https://api.example.com:443/v1/users?id=123#results",
"http://192.168.1.1:3000/admin",
"ws://websocket.example.com/socket",
]
test_urls.each do |url_str|
puts "-" * 40
puts "Input: #{url_str}"
url = parse_url(url_str)
if url
puts " Protocol: #{url.protocol}"
puts " Host: #{url.host}"
puts " Port: #{url.port || '(default)'}"
puts " Path: #{url.path || '/'}"
puts " Query: #{url.query || '(none)'}"
puts " Fragment: #{url.fragment || '(none)'}"
puts " Reconstructed: #{url}"
end
puts
end
end