import os import sys import re import json import traceback from collections import defaultdict try: from urllib.request import urlopen, Request from urllib.parse import unquote, quote except ImportError: from urllib2 import urlopen, Request from urllib import unquote, quote sys.path.insert(0, os.path.dirname(__file__)) STOP_WORDS = set([ "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can", "could", "did", "do", "does", "doing", "down", "during", "each", "few", "for", "from", "further", "had", "has", "have", "having", "he", "her", "here", "hers", "herself", "him", "himself", "his", "how", "i", "if", "in", "into", "is", "it", "its", "itself", "just", "me", "more", "most", "my", "myself", "no", "nor", "not", "now", "of", "off", "on", "once", "only", "or", "other", "our", "ours", "ourselves", "out", "over", "own", "s", "same", "she", "should", "so", "some", "such", "t", "than", "that", "the", "their", "theirs", "them", "themselves", "then", "there", "these", "they", "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "we", "were", "what", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "you", "your", "yours", "yourself", "yourselves" ]) def strip_tags(html): # Remove script, style, and structural elements (header, footer, nav) entirely html = re.sub(r'<(script|style|noscript|meta|nav|header|footer|aside)[^>]*>.*?', ' ', html, flags=re.IGNORECASE|re.DOTALL) html = re.sub(r'<[^>]+>', ' ', html) html = re.sub(r'&[a-z0-9]+;', ' ', html, flags=re.IGNORECASE) return html def is_number(s): try: float(s) if '.' in s else int(s) return True except ValueError: return False def generate_base_keywords(text, top_n=5): # RAKE Algorithm sentence_delimiters = re.compile(r'[.!?,;:\t\\\"\\(\\)\\\'\u2019\u2013]|\\s\\-\\s') sentences = sentence_delimiters.split(text) phrase_list = [] for s in sentences: s = s.strip().lower() if not s: continue words = re.findall(r'\b[a-z0-9]+\b', s) current_phrase = [] for w in words: if w in STOP_WORDS or is_number(w) or len(w) <= 2: if current_phrase: # Ignore massive chunks of text (like navigation menus) if len(current_phrase) <= 3: phrase_list.append(" ".join(current_phrase)) current_phrase = [] else: current_phrase.append(w) if current_phrase: if len(current_phrase) <= 3: phrase_list.append(" ".join(current_phrase)) word_freq = defaultdict(int) word_degree = defaultdict(int) for phrase in phrase_list: words = phrase.split() length = len(words) for w in words: word_freq[w] += 1 word_degree[w] += length word_score = {} for w in word_freq: word_score[w] = word_degree[w] / float(word_freq[w]) phrase_scores = {} for phrase in phrase_list: if phrase not in phrase_scores: score = 0.0 for w in phrase.split(): score += word_score[w] phrase_scores[phrase] = score sorted_phrases = sorted(phrase_scores.items(), key=lambda x: x[1], reverse=True) unique_keywords = [] seen = set() for phrase, score in sorted_phrases: if phrase not in seen: unique_keywords.append(phrase) seen.add(phrase) if len(unique_keywords) >= top_n: break return unique_keywords def get_synonyms(keyword, max_results=3): """ Fetches synonyms and related intent words using the free Datamuse API. Requires ZERO API keys and is extremely fast. """ try: # 'ml' stands for 'means like' - grabs synonyms and contextually related words url = "https://api.datamuse.com/words?ml=" + quote(keyword) + "&max=" + str(max_results) headers = {'User-Agent': 'Mozilla/5.0'} req = Request(url, headers=headers) response = urlopen(req, timeout=5) data = response.read() if sys.version_info[0] >= 3: data = data.decode('utf-8', errors='ignore') results = json.loads(data) synonyms = [item['word'] for item in results] return synonyms except Exception: return [] def extract_and_expand_keywords(text): # 1. Get the top exact keywords from the text base_keywords = generate_base_keywords(text, top_n=5) # 2. Expand with synonyms for search intent final_keywords = set(base_keywords) for kw in base_keywords: synonyms = get_synonyms(kw, max_results=3) for syn in synonyms: # Only add synonyms that are distinct from the original text (broadens search) final_keywords.add(syn) return list(final_keywords) def parse_query_string(query_string): params = {} if not query_string: return params pairs = query_string.split('&') for p in pairs: if '=' in p: k, v = p.split('=', 1) params[k] = unquote(v) return params def application(environ, start_response): try: query_string = environ.get('QUERY_STRING', '') params = parse_query_string(query_string) url = params.get('url') if url: if not url.startswith('http://') and not url.startswith('https://'): url = 'https://' + url headers = {'User-Agent': 'Mozilla/5.0'} req = Request(url, headers=headers) response = urlopen(req, timeout=10) html_content = response.read() if sys.version_info[0] >= 3: try: html_content = html_content.decode('utf-8', errors='ignore') except Exception: pass text = strip_tags(html_content) # Extract keywords AND generate synonyms! keywords = extract_and_expand_keywords(text) response_body = json.dumps(keywords) status = '200 OK' else: response_body = json.dumps({"error": "Missing 'url' parameter. Example: ?url=https://example.com"}) status = '400 Bad Request' response_headers = [ ('Content-Type', 'application/json'), ('Access-Control-Allow-Origin', '*'), ('Content-Length', str(len(response_body))) ] start_response(status, response_headers) if sys.version_info[0] >= 3: return [response_body.encode('utf-8')] else: return [response_body] except Exception as e: start_response('200 OK', [('Content-Type', 'text/plain')]) error_msg = "Python Application Error:\n\n" + traceback.format_exc() + "\n\nPython Version: " + sys.version if sys.version_info[0] >= 3: return [error_msg.encode('utf-8')] else: return [error_msg]