Bug Bounty: Finding the testing focus by filtering for the amount of URL paths

Vulnerabilities are a minority of events, so the focus should be on identifying the top N assets from any dimension that possible have these vulnerabilities.

Then I created a simple tool called spurls with AI. Spurls, which stands for “special URLs”, identifies a list of unique URLs by constructing a URL list into a tree structure and then filtering based on the number of nodes. The source code will be placed at the end.

Usage: ./spurls <input_file> [root_threshold] [leaf_threshold] [child_threshold] [descendant_threshold]
Arguments:
 input_file: Path to the text file containing a list of URLs (required)
 root_threshold: Threshold for the number of children of a root node (optional, default 2000)
 leaf_threshold: Threshold for the number of leaf nodes (optional, default 100)
 child_threshold: Threshold for the number of children of a node (optional, default 500)
 descendant_threshold: Threshold for the total number of descendant nodes (optional, default 2000)

Besides the root_threshold parameter, all other threshold parameters apply to non-root nodes.

./spurls all_urls.txt 2000 5 100 2000 | grep -vP '\.(js|json|xml|png|ico|jpg|gif|woff2|css|otf)$' | grep -v '^$' > spurls.txt

Usage 1: Filter top N domains

cat spurls.txt | cut -d/ -f3 | grep -v '^$' | sort | uniq -c | sort -rn > spurls_domain.txt

Usage 2: Filter top N URL paths

cat spurls.txt | cut -d/ -f4- | grep -v '^$' | sort | uniq -c | sort -n | awk '{printf "%s@_@/%s\n", $1, $2}' > spurls_path.txt

Usage 3: Filter the original URLs corresponding to the top N URL paths(Convert the path to a regular expression and then match it against the original URL list. If you have a better method, please reply directly below the article.)

cat spurls_path.txt | awk -F '@_@' '{print $2}' | grep -v '^$' | grep -vP "(\"|')" | sed -e 's/[][\.\*\^\$\(\)\{\}\?\+\|]/\\\\&/g' -e 's/^/\^https?:\/\/\[\^\/\]\*/' -e 's/$/\(\$\|\\\\?\)/' | xargs -I {} grep -P '{}' all_urls.txt >> spurls_origin.txt

In my own real-world testing, I’ve found that only when the URL data collection is comprehensive can effectively filter out key testing targets.

main.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include "url_tree.h"

#define MAX_URL_LENGTH 2048
#define MAX_LINE_LENGTH 4096

// Default thresholds
#define DEFAULT_ROOT_THRESHOLD 2000
#define DEFAULT_LEAF_THRESHOLD 100
#define DEFAULT_CHILD_THRESHOLD 500
#define DEFAULT_DESCENDANT_THRESHOLD 2000

void print_usage(const char *program_name) {
    printf("Usage: %s <input_file> [root_threshold] [leaf_threshold] [child_threshold] [descendant_threshold]\n", program_name);
    printf("Arguments:\n");
    printf("  input_file: Path to the text file containing a list of URLs (required)\n");
    printf("  root_threshold: Threshold for the number of children of a root node (optional, default %d)\n", DEFAULT_ROOT_THRESHOLD);
    printf("  leaf_threshold: Threshold for the number of leaf nodes (optional, default %d)\n", DEFAULT_LEAF_THRESHOLD);
    printf("  child_threshold: Threshold for the number of children of a node (optional, default %d)\n", DEFAULT_CHILD_THRESHOLD);
    printf("  descendant_threshold: Threshold for the total number of descendant nodes (optional, default %d)\n", DEFAULT_DESCENDANT_THRESHOLD);
    printf("\nExamples:\n");
    printf("  %s urls.txt 2000 5 100 2000 | sort | uniq\n", program_name);
}

// Parse command-line arguments
int parse_args(int argc, char *argv[], int *root_threshold, int *leaf_threshold,
               int *child_threshold, int *descendant_threshold) {
    // Check for required arguments
    if (argc < 2) {
        return 0;
    }

    // Set default values
    *root_threshold = DEFAULT_ROOT_THRESHOLD;
    *leaf_threshold = DEFAULT_LEAF_THRESHOLD;
    *child_threshold = DEFAULT_CHILD_THRESHOLD;
    *descendant_threshold = DEFAULT_DESCENDANT_THRESHOLD;

    // Parse optional arguments
    if (argc > 2) *root_threshold = atoi(argv[2]);
    if (argc > 3) *leaf_threshold = atoi(argv[3]);
    if (argc > 4) *child_threshold = atoi(argv[4]);
    if (argc > 5) *descendant_threshold = atoi(argv[5]);

    return 1;
}

int main(int argc, char *argv[]) {
    int root_threshold, leaf_threshold, child_threshold, descendant_threshold;

    // Parse command-line arguments
    if (!parse_args(argc, argv, &root_threshold, &leaf_threshold,
                    &child_threshold, &descendant_threshold)) {
        print_usage(argv[0]);
        return 1;
    }

    const char *input_file = argv[1];

    // Open input file
    FILE *fp = fopen(input_file, "r");
    if (!fp) {
        printf("Error: Cannot open input file %s\n", input_file);
        return 1;
    }

    // Create URL tree
    URLTree *tree = create_url_tree();
    char line[MAX_LINE_LENGTH];

    // Read and process URLs
    while (fgets(line, sizeof(line), fp)) {
        // Remove newline character
        line[strcspn(line, "\n")] = 0;

        // Skip empty lines
        if (strlen(line) == 0) continue;

        // Add URL to the tree
        add_url_to_tree(tree, line);
    }
    fclose(fp);

    // Filter and output results to standard output
    filter_and_output_urls(tree, root_threshold, leaf_threshold, child_threshold,
                           descendant_threshold, stdout);

    // Free resources
    free_url_tree(tree);

    return 0;
}

url_tree.h

#ifndef URL_TREE_H
#define URL_TREE_H

#include <stdio.h>

// URL node structure
typedef struct URLNode {
    char *name;               // Node name
    char *full_url;           // Full URL
    struct URLNode **children;  // Array of child nodes
    int children_count;       // Number of child nodes
    int children_capacity;      // Capacity of child nodes array
} URLNode;

// URL tree structure
typedef struct {
    URLNode **hosts;          // Array of host nodes
    int hosts_count;          // Number of host nodes
    int hosts_capacity;       // Capacity of host nodes array
} URLTree;

// Create a new URL tree
URLTree* create_url_tree();

// Free URL tree resources
void free_url_tree(URLTree *tree);

// Add a URL to the tree
void add_url_to_tree(URLTree *tree, const char *url);

// Filter and output URLs that meet the criteria
void filter_and_output_urls(URLTree *tree, int root_threshold, int leaf_threshold,
                            int child_threshold, int descendant_threshold, FILE *output);

// Parse a URL to extract the host and path
int parse_url(const char *url, char **host, char **path);

// Create a new URL node
URLNode* create_url_node(const char *name, const char *full_url);

// Free URL node resources
void free_url_node(URLNode *node);

// Add a child node
void add_child_node(URLNode *parent, URLNode *child);

// Get the full path of a node
char* get_node_path(URLNode *node);

#endif // URL_TREE_H

url_tree.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include "url_tree.h"

#define INITIAL_CAPACITY 16

// Create a new URL tree
URLTree* create_url_tree() {
    URLTree *tree = (URLTree*)malloc(sizeof(URLTree));
    if (!tree) return NULL;

    tree->hosts = (URLNode**)malloc(sizeof(URLNode*) * INITIAL_CAPACITY);
    if (!tree->hosts) {
        free(tree);
        return NULL;
    }

    tree->hosts_count = 0;
    tree->hosts_capacity = INITIAL_CAPACITY;
    return tree;
}

// Create a new URL node
URLNode* create_url_node(const char *name, const char *full_url) {
    URLNode *node = (URLNode*)malloc(sizeof(URLNode));
    if (!node) return NULL;

    node->name = strdup(name);
    node->full_url = strdup(full_url);
    node->children = (URLNode**)malloc(sizeof(URLNode*) * INITIAL_CAPACITY);
    node->children_count = 0;
    node->children_capacity = INITIAL_CAPACITY;

    return node;
}

// Free URL node resources
void free_url_node(URLNode *node) {
    if (!node) return;

    for (int i = 0; i < node->children_count; i++) {
        free_url_node(node->children[i]);
    }

    free(node->name);
    free(node->full_url);
    free(node->children);
    free(node);
}

// Free URL tree resources
void free_url_tree(URLTree *tree) {
    if (!tree) return;

    for (int i = 0; i < tree->hosts_count; i++) {
        free_url_node(tree->hosts[i]);
    }

    free(tree->hosts);
    free(tree);
}

// Add a child node
void add_child_node(URLNode *parent, URLNode *child) {
    if (parent->children_count >= parent->children_capacity) {
        parent->children_capacity *= 2;
        parent->children = (URLNode**)realloc(parent->children,
            sizeof(URLNode*) * parent->children_capacity);
    }
    parent->children[parent->children_count++] = child;
}

// Normalize URL (remove extra slashes, spaces, query parameters, and fragments)
char* normalize_url(const char* url) {
    char* normalized = strdup(url);
    char* p = normalized;
    char* q = normalized;

    // Skip leading whitespace
    while (isspace(*p)) p++;

    // Copy and normalize
    while (*p) {
        // Stop at query parameters or fragments
        if (*p == '?' || *p == '#') {
            break;
        }

        // Skip consecutive slashes, but keep the double slashes for the protocol part
        if (*p == '/' && *(p+1) == '/') {
            // Check if it's the double slash of the protocol
            if (p > normalized && *(p-1) == ':') {
                *q++ = *p++;  // Keep the first slash
                *q++ = *p++;  // Keep the second slash
                continue;
            }
            p++;
            continue;
        }
        // Skip trailing slashes
        if (*p == '/' && *(p+1) == '\0') {
            p++;
            continue;
        }
        *q++ = *p++;
    }
    *q = '\0';

    return normalized;
}

// Parse URL to extract the host and path
int parse_url(const char *url, char **host, char **path) {
    const char *protocol_end = strstr(url, "://");
    if (!protocol_end) return 0;

    const char *host_start = protocol_end + 3;
    const char *path_start = strchr(host_start, '/');
    const char *query_start = strchr(host_start, '?');
    const char *fragment_start = strchr(host_start, '#');

    // Determine the end of the path (before query parameters or fragments)
    const char *path_end = host_start + strlen(host_start);
    if (query_start && query_start < path_end) {
        path_end = query_start;
    }
    if (fragment_start && fragment_start < path_end) {
        path_end = fragment_start;
    }

    if (!path_start) {
        *host = strdup(host_start);
        *path = strdup("/");
        return 1;
    }

    int host_len = path_start - host_start;
    *host = (char*)malloc(host_len + 1);
    strncpy(*host, host_start, host_len);
    (*host)[host_len] = '\0';

    int path_len = path_end - path_start;
    *path = (char*)malloc(path_len + 1);
    strncpy(*path, path_start, path_len);
    (*path)[path_len] = '\0';

    return 1;
}

// Find or create a child node
URLNode* find_or_create_child(URLNode *parent, const char *name, const char *full_url) {
    // Normalize URL
    char* normalized_url = normalize_url(full_url);

    // First, check if a child node with the same name already exists
    for (int i = 0; i < parent->children_count; i++) {
        if (strcmp(parent->children[i]->name, name) == 0) {
            // If a node with the same name is found, update its full URL (keep the latest URL)
            free(parent->children[i]->full_url);
            parent->children[i]->full_url = normalized_url;
            return parent->children[i];
        }
    }

    // If no node with the same name is found, create a new node
    URLNode *new_node = create_url_node(name, normalized_url);
    if (!new_node) {
        free(normalized_url);
        return NULL;
    }

    // Add the new node to the parent
    if (parent->children_count >= parent->children_capacity) {
        parent->children_capacity *= 2;
        parent->children = (URLNode**)realloc(parent->children,
            sizeof(URLNode*) * parent->children_capacity);
        if (!parent->children) {
            free_url_node(new_node);
            return NULL;
        }
    }
    parent->children[parent->children_count++] = new_node;
    return new_node;
}

// Add a URL to the tree
void add_url_to_tree(URLTree *tree, const char *url) {
    char *host, *path;
    if (!parse_url(url, &host, &path)) {
        return;
    }

    // Find or create the host node
    URLNode *host_node = NULL;
    for (int i = 0; i < tree->hosts_count; i++) {
        if (strcmp(tree->hosts[i]->name, host) == 0) {
            host_node = tree->hosts[i];
            break;
        }
    }

    if (!host_node) {
        if (tree->hosts_count >= tree->hosts_capacity) {
            tree->hosts_capacity *= 2;
            tree->hosts = (URLNode**)realloc(tree->hosts,
                sizeof(URLNode*) * tree->hosts_capacity);
        }
        host_node = create_url_node(host, url);
        tree->hosts[tree->hosts_count++] = host_node;
    }

    // Process the path
    char *path_copy = strdup(path);
    char *token = strtok(path_copy, "/");
    URLNode *current = host_node;

    while (token) {
        current = find_or_create_child(current, token, url);
        token = strtok(NULL, "/");
    }

    free(path_copy);
    free(host);
    free(path);
}

// Count the total number of all descendant nodes
int count_descendants(URLNode *node) {
    int count = node->children_count;
    for (int i = 0; i < node->children_count; i++) {
        count += count_descendants(node->children[i]);
    }
    return count;
}

// Check if a node is a leaf node (has no children)
int is_leaf_node(URLNode *node) {
    return node->children_count == 0;
}

// Recursively check if the node meets the criteria
void check_node(URLNode *node, int current_level, int root_threshold, int leaf_threshold,
                int child_threshold, int descendant_threshold, FILE *output, int is_root) {
    // If it's a root node, check the number of its children
    if (is_root && node->children_count > root_threshold) {
        return; // Skip all children of this root node
    }

    // If it's not a root node, check various conditions
    if (!is_root) {
        // Condition 1: If not a root node, and children count > leaf_threshold, and has no grandchildren, skip children
        if (node->children_count > leaf_threshold) {
            int has_grandchildren = 0;
            for (int i = 0; i < node->children_count; i++) {
                if (!is_leaf_node(node->children[i])) {
                    has_grandchildren = 1;
                    break;
                }
            }
            if (!has_grandchildren) {
                return; // Skip all children of this node
            }
        }

        // Condition 2: If not a root node, and children count > child_threshold, skip children
        if (node->children_count > child_threshold) {
            return; // Skip all children of this node
        }

        // Condition 3: If the current node is not a root node and the total number of all descendants exceeds descendant_threshold, skip subsequent children
        if (count_descendants(node) > descendant_threshold) {
            return; // Skip all children of this node
        }
    }

    // If it's a leaf node (has no children), output the URL
    if (node->children_count == 0) {
        char* normalized_url = normalize_url(node->full_url);
        fprintf(output, "%s\n", normalized_url);
        free(normalized_url);
        return;
    }

    // Recursively process child nodes
    for (int i = 0; i < node->children_count; i++) {
        check_node(node->children[i], current_level + 1, root_threshold, leaf_threshold,
                     child_threshold, descendant_threshold, output, 0);
    }
}

// Filter and output URLs that meet the criteria
void filter_and_output_urls(URLTree *tree, int root_threshold, int leaf_threshold,
                            int child_threshold, int descendant_threshold, FILE *output) {
    for (int i = 0; i < tree->hosts_count; i++) {
        check_node(tree->hosts[i], 0, root_threshold, leaf_threshold, child_threshold,
                     descendant_threshold, output, 1); // 1 indicates it's a root node
    }
}