🧠 NLP & Text Processing

import re, string

text = '  Hello, World! This is NLP 101... Check <b>this</b> out!  '

# Lowercase
clean = text.lower()
# Strip HTML tags
clean = re.sub(r'<[^>]+>', '', clean)
# Remove punctuation
clean = clean.translate(str.maketrans('', '', string.punctuation))
# Collapse whitespace
clean = ' '.join(clean.split())
print(repr(clean))
# Output: 'hello world this is nlp 101 check this out'

import re

def clean_text(text):
    text = re.sub(r'http\S+|www\.\S+', '', text)   # URLs
    text = re.sub(r'@\w+', '', text)                 # @mentions
    text = re.sub(r'#\w+', '', text)                 # hashtags
    text = re.sub(r'[^\w\s]', '', text)             # punctuation
    text = re.sub(r'\d+', '', text)                  # digits
    text = re.sub(r'\s+', ' ', text).strip()         # whitespace
    return text.lower()

tweet = 'Check out https://example.com #NLP @user! 123 Great stuff!!!'
print(clean_text(tweet))
# Output: 'check out  great stuff'

import unicodedata

text = 'Café naïve résumé — quotes \u2018smart\u2019'

# Normalize to ASCII (strip accents)
def to_ascii(text):
    nfkd = unicodedata.normalize('NFKD', text)
    return ''.join(c for c in nfkd if not unicodedata.combining(c))

# Normalize smart quotes to straight
def fix_quotes(text):
    replacements = {\u2018: "'", \u2019: "'", \u201c: '"', \u201d: '"',
                    \u2013: '-', \u2014: '-'}
    return ''.join(replacements.get(c, c) for c in text)

print(to_ascii(text))
print(fix_quotes(text))

import re, string
from typing import List, Callable

def make_pipeline(*fns: Callable) -> Callable:
    def pipeline(text: str) -> str:
        for fn in fns:
            text = fn(text)
        return text
    return pipeline

lowercase       = str.lower
remove_urls     = lambda t: re.sub(r'http\S+', '', t)
remove_punct    = lambda t: t.translate(str.maketrans('', '', string.punctuation))
collapse_spaces = lambda t: ' '.join(t.split())

clean = make_pipeline(lowercase, remove_urls, remove_punct, collapse_spaces)

texts = ['Visit https://ai.com for more!', 'Hello, World!!', '  PYTHON  NLP  ']
print([clean(t) for t in texts])

import nltk
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
from nltk.tokenize import word_tokenize, sent_tokenize

text = 'Dr. Smith said NLP is fun. It really is! Don\'t you think?'

sentences = sent_tokenize(text)
print('Sentences:', sentences)

words = word_tokenize(text)
print('Words:', words)

import nltk
nltk.download('stopwords', quiet=True)
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

stop_words = set(stopwords.words('english'))

text = 'The quick brown fox jumps over the lazy dog'
tokens = word_tokenize(text.lower())
filtered = [w for w in tokens if w.isalpha() and w not in stop_words]

print('Original tokens:', tokens)
print('Filtered:', filtered)

try:
    import spacy
    nlp = spacy.load('en_core_web_sm')
    text = 'Apple is looking at buying U.K. startup for $1 billion.'
    doc = nlp(text)
    tokens = [(t.text, t.pos_, t.is_stop) for t in doc]
    print('(token, POS, is_stop):')
    for tok in tokens:
        print(tok)
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

try:
    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')

    text = 'Tokenization handles out-of-vocabulary words cleverly.'
    tokens = tokenizer.tokenize(text)
    ids = tokenizer.encode(text)

    print('Subword tokens:', tokens)
    print('Token IDs:', ids)
    print('Decoded:', tokenizer.decode(ids))
except ImportError:
    print('pip install transformers')

import nltk
from nltk.stem import PorterStemmer, SnowballStemmer

porter = PorterStemmer()
snowball = SnowballStemmer('english')

words = ['running', 'flies', 'happily', 'studies', 'beautiful', 'caring']

print(f'{'Word':<15} {'Porter':<15} {'Snowball':<15}')
for w in words:
    print(f'{w:<15} {porter.stem(w):<15} {snowball.stem(w):<15}')

import nltk
nltk.download('wordnet', quiet=True)
nltk.download('omw-1.4', quiet=True)
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet

lemmatizer = WordNetLemmatizer()

# POS matters: 'better' as ADJ -> 'good', as VERB -> 'better'
examples = [
    ('better',  'a'),   # adjective
    ('running', 'v'),   # verb
    ('geese',   'n'),   # noun
    ('happily', 'r'),   # adverb
]
for word, pos in examples:
    lem = lemmatizer.lemmatize(word, pos=pos)
    print(f'{word} ({pos}) -> {lem}')

try:
    import spacy
    nlp = spacy.load('en_core_web_sm')

    text = 'The children were running and the geese were flying'
    doc = nlp(text)

    print(f'{'Token':<15} {'Lemma':<15} {'POS':<10}')
    for token in doc:
        print(f'{token.text:<15} {token.lemma_:<15} {token.pos_:<10}')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

import nltk
nltk.download('wordnet', quiet=True)
nltk.download('omw-1.4', quiet=True)
from nltk.stem import PorterStemmer, WordNetLemmatizer

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

words = ['studies', 'studying', 'better', 'wolves', 'corpora', 'matrices']

print(f'{'Word':<12} {'Stem':<12} {'Lemma (n)':<12}')
for w in words:
    stem = stemmer.stem(w)
    lemma = lemmatizer.lemmatize(w, pos='n')
    print(f'{w:<12} {stem:<12} {lemma:<12}')

try:
    import spacy
    nlp = spacy.load('en_core_web_sm')

    text = 'Apple Inc. was founded by Steve Jobs in Cupertino on April 1, 1976.'
    doc = nlp(text)

    print('Entities found:')
    for ent in doc.ents:
        print(f'  {ent.text:<25} {ent.label_:<12} {spacy.explain(ent.label_)}')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

import nltk
nltk.download('averaged_perceptron_tagger', quiet=True)
nltk.download('maxent_ne_chunker', quiet=True)
nltk.download('words', quiet=True)
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
nltk.download('averaged_perceptron_tagger_eng', quiet=True)
from nltk import word_tokenize, pos_tag, ne_chunk
from nltk.tree import Tree

text = 'Barack Obama served as the 44th President of the United States.'
tokens = word_tokenize(text)
tagged = pos_tag(tokens)
chunks = ne_chunk(tagged)

for subtree in chunks:
    if isinstance(subtree, Tree):
        entity = ' '.join(word for word, tag in subtree.leaves())
        print(f'{entity}: {subtree.label()}')

try:
    import spacy
    from spacy import displacy
    nlp = spacy.load('en_core_web_sm')

    text = 'Elon Musk founded SpaceX in Hawthorne, California in 2002.'
    doc = nlp(text)

    # In a Jupyter notebook this renders inline:
    # displacy.render(doc, style='ent')

    # Save to HTML:
    html = displacy.render(doc, style='ent', page=True)
    print(html[:200], '...')  # Show snippet
    print('\nEntities:', [(e.text, e.label_) for e in doc.ents])
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

try:
    import spacy
    nlp = spacy.load('en_core_web_sm')

    # Add custom entity patterns
    ruler = nlp.add_pipe('entity_ruler', before='ner')
    patterns = [
        {'label': 'TECH_STACK', 'pattern': 'Python'},
        {'label': 'TECH_STACK', 'pattern': 'TensorFlow'},
        {'label': 'TECH_STACK', 'pattern': [{'LOWER': 'scikit'}, {'LOWER': '-'}, {'LOWER': 'learn'}]},
    ]
    ruler.add_patterns(patterns)

    doc = nlp('We use Python and TensorFlow with scikit-learn for ML.')
    for ent in doc.ents:
        print(f'{ent.text}: {ent.label_}')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

import nltk
nltk.download('vader_lexicon', quiet=True)
from nltk.sentiment.vader import SentimentIntensityAnalyzer

sia = SentimentIntensityAnalyzer()

texts = [
    'The food was absolutely amazing and the service was great!',
    'Terrible experience. Never going back.',
    'The product arrived on time.',
    'Not bad, but could be better.',
]

for text in texts:
    scores = sia.polarity_scores(text)
    label = 'POSITIVE' if scores['compound'] > 0.05 else 'NEGATIVE' if scores['compound'] < -0.05 else 'NEUTRAL'
    print(f'{label}: {text[:40]:<40} | compound={scores["compound"]:.3f}')

try:
    from textblob import TextBlob

    reviews = [
        'Absolutely love this product! Best purchase ever.',
        'Disappointed. Quality is poor and shipping was slow.',
        'It is okay. Nothing special.',
    ]

    for review in reviews:
        blob = TextBlob(review)
        pol = blob.sentiment.polarity       # -1 to 1
        sub = blob.sentiment.subjectivity   # 0 (objective) to 1 (subjective)
        print(f'Polarity: {pol:+.2f}  Subjectivity: {sub:.2f}  | {review[:40]}')
except ImportError:
    print('pip install textblob')

try:
    from transformers import pipeline

    sentiment = pipeline('sentiment-analysis',
                         model='distilbert-base-uncased-finetuned-sst-2-english')

    texts = [
        'I love this movie so much!',
        'This is the worst product I have ever bought.',
        'The package arrived in reasonable time.',
    ]
    results = sentiment(texts)
    for text, result in zip(texts, results):
        print(f'{result["label"]:<10} ({result["score"]:.3f}): {text}')
except ImportError:
    print('pip install transformers torch')

import nltk
nltk.download('vader_lexicon', quiet=True)
from nltk.sentiment.vader import SentimentIntensityAnalyzer

sia = SentimentIntensityAnalyzer()

ASPECTS = {
    'battery': ['battery', 'charge', 'charging', 'power'],
    'camera':  ['camera', 'photo', 'picture', 'image'],
    'screen':  ['screen', 'display', 'resolution'],
}

def aspect_sentiment(review):
    results = {}
    sentences = nltk.sent_tokenize(review)
    for aspect, keywords in ASPECTS.items():
        relevant = [s for s in sentences if any(k in s.lower() for k in keywords)]
        if relevant:
            scores = [sia.polarity_scores(s)['compound'] for s in relevant]
            results[aspect] = sum(scores) / len(scores)
    return results

review = 'Battery life is excellent! But the camera quality is disappointing. The screen is stunning.'
print(aspect_sentiment(review))

from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

corpus = [
    'the cat sat on the mat',
    'the dog lay on the rug',
    'cats and dogs are both great pets',
]

vec = TfidfVectorizer(stop_words='english')
X = vec.fit_transform(corpus)

df = pd.DataFrame(X.toarray(), columns=vec.get_feature_names_out())
print(df.round(3))

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

docs = [
    'Python is great for data science',
    'Data science uses Python and R',
    'I love cooking Italian food',
    'Machine learning with Python and sklearn',
]

vec = TfidfVectorizer(stop_words='english')
X = vec.fit_transform(docs)
sim = cosine_similarity(X)

print('Similarity matrix:')
for i, row in enumerate(sim):
    print(f'Doc {i}: {[f"{v:.2f}" for v in row]}')

try:
    from gensim.models import Word2Vec
    import nltk
    nltk.download('punkt', quiet=True)
    nltk.download('punkt_tab', quiet=True)
    from nltk.tokenize import word_tokenize

    sentences = [
        'king is a powerful man',
        'queen is a powerful woman',
        'boy is a young man',
        'girl is a young woman',
    ]
    tokenized = [word_tokenize(s) for s in sentences]
    model = Word2Vec(tokenized, vector_size=50, window=3, min_count=1, epochs=100)

    # Classic word vector arithmetic
    result = model.wv.most_similar(positive=['king', 'woman'], negative=['man'], topn=3)
    print('king + woman - man =', result)
    print('Similarity(king, queen):', model.wv.similarity('king', 'queen'))
except ImportError:
    print('pip install gensim')

try:
    from sentence_transformers import SentenceTransformer
    from sklearn.metrics.pairwise import cosine_similarity

    model = SentenceTransformer('all-MiniLM-L6-v2')

    sentences = [
        'A man is playing guitar.',
        'Someone is strumming a musical instrument.',
        'A cat is sitting on the couch.',
    ]

    embeddings = model.encode(sentences)
    sim = cosine_similarity(embeddings)

    print('Semantic similarity:')
    for i in range(len(sentences)):
        for j in range(i+1, len(sentences)):
            print(f'  [{i}] vs [{j}]: {sim[i,j]:.3f}')
except ImportError:
    print('pip install sentence-transformers')

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import numpy as np

docs = [
    'baseball team pitcher bat home run stadium',
    'football touchdown quarterback field goal referee',
    'stock market shares dividends portfolio investor',
    'bitcoin ethereum blockchain cryptocurrency wallet',
    'machine learning neural network deep learning AI',
    'python data science pandas numpy statistics',
]

vec = CountVectorizer(stop_words='english')
X = vec.fit_transform(docs)

lda = LatentDirichletAllocation(n_components=3, random_state=42)
lda.fit(X)

feature_names = vec.get_feature_names_out()
for i, topic in enumerate(lda.components_):
    top_words = [feature_names[j] for j in topic.argsort()[-6:][::-1]]
    print(f'Topic {i}: {top_words}')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF

docs = [
    'health fitness exercise gym workout training',
    'diet nutrition calories protein weight loss',
    'travel vacation flight hotel beach tourism',
    'passport visa travel destination adventure explore',
    'cooking recipe chef kitchen ingredients bake',
]

vec = TfidfVectorizer(stop_words='english', max_features=50)
X = vec.fit_transform(docs)

nmf = NMF(n_components=3, random_state=42)
W = nmf.fit_transform(X)
H = nmf.components_

feature_names = vec.get_feature_names_out()
for i, topic in enumerate(H):
    top_words = [feature_names[j] for j in topic.argsort()[-5:][::-1]]
    print(f'Topic {i}: {top_words}')

print('\nDoc-topic assignments (W):')
for i, row in enumerate(W):
    print(f'Doc {i}: topic {row.argmax()}')

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import numpy as np

# Synthetic corpus
docs = (['python data science machine learning'] * 5 +
        ['football soccer game stadium team'] * 5 +
        ['stock market finance investment portfolio'] * 5)

vec = CountVectorizer(stop_words='english')
X = vec.fit_transform(docs)

perplexities = []
k_range = range(2, 7)
for k in k_range:
    lda = LatentDirichletAllocation(n_components=k, random_state=42, max_iter=20)
    lda.fit(X)
    perplexities.append(lda.perplexity(X))
    print(f'k={k}: perplexity={lda.perplexity(X):.2f}')

best_k = k_range[np.argmin(perplexities)]
print(f'Best k: {best_k}')

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import numpy as np

train_docs = [
    'python programming code software developer',
    'machine learning model training dataset',
    'basketball players championship game team',
    'tennis grand slam tournament court player',
]
TOPIC_LABELS = {0: 'Technology', 1: 'Sports'}  # manual labels

vec = CountVectorizer(stop_words='english')
X_train = vec.fit_transform(train_docs)

lda = LatentDirichletAllocation(n_components=2, random_state=42)
lda.fit(X_train)

new_docs = [
    'neural network deep learning GPU training',
    'football touchdown quarterback Super Bowl',
]
X_new = vec.transform(new_docs)
topic_dist = lda.transform(X_new)
for doc, dist in zip(new_docs, topic_dist):
    label = TOPIC_LABELS.get(dist.argmax(), f'Topic {dist.argmax()}')
    print(f'{doc[:40]} -> {label} ({dist.max():.2f})')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# Minimal spam dataset
texts = [
    'Win a FREE iPhone now! Click here!!!', 'URGENT: You have won $1000',
    'Claim your prize today, limited offer!', 'Hot singles in your area',
    'Meeting at 3pm in the conference room', 'Can you review my pull request?',
    'Lunch tomorrow? Let me know.', 'Project deadline is Friday.',
    'Budget report attached for your review', 'Hi, are you free this afternoon?',
]
labels = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0]  # 1=spam, 0=ham

X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=0.3, random_state=42)

clf = Pipeline([('tfidf', TfidfVectorizer()), ('nb', MultinomialNB())])
clf.fit(X_train, y_train)
print(classification_report(y_test, clf.predict(X_test), target_names=['ham', 'spam']))

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
import numpy as np

texts = [
    'The stock market crashed today', 'Interest rates affect mortgages',
    'Champions League final tonight', 'NBA playoffs heating up',
    'New deep learning model released', 'Python 4.0 features announced',
    'Election results pending', 'Senate votes on new bill',
]
labels = ['finance','finance','sports','sports','tech','tech','politics','politics']

pipe = Pipeline([
    ('tfidf', TfidfVectorizer(ngram_range=(1, 2))),
    ('lr',   LogisticRegression(max_iter=200))
])
scores = cross_val_score(pipe, texts, labels, cv=2, scoring='accuracy')
print(f'CV accuracy: {scores.mean():.2f} ± {scores.std():.2f}')

pipe.fit(texts, labels)
print(pipe.predict(['Bitcoin surges to all-time high']))

try:
    from transformers import pipeline

    classifier = pipeline('zero-shot-classification',
                          model='facebook/bart-large-mnli')

    texts = [
        'The Federal Reserve raises interest rates by 25 basis points.',
        'Team wins championship after dramatic overtime goal.',
        'New AI model achieves human-level performance on benchmark.',
    ]
    candidate_labels = ['finance', 'sports', 'technology', 'politics']

    for text in texts:
        result = classifier(text, candidate_labels)
        print(f'{result["labels"][0]:12} ({result["scores"][0]:.2f}): {text[:50]}')
except ImportError:
    print('pip install transformers torch')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
import numpy as np

texts = [
    'stock market bull bear portfolio dividends',
    'goal touchdown home run stadium championship',
    'algorithm neural network training dataset model',
    'vote senator election campaign policy',
] * 3
labels = ['finance','sports','tech','politics'] * 3

pipe = Pipeline([('tfidf', TfidfVectorizer()), ('lr', LogisticRegression(max_iter=200))])
pipe.fit(texts, labels)

feature_names = pipe['tfidf'].get_feature_names_out()
classes = pipe['lr'].classes_

for cls, coef in zip(classes, pipe['lr'].coef_):
    top = [feature_names[i] for i in coef.argsort()[-5:][::-1]]
    print(f'{cls}: {top}')

try:
    from transformers import AutoTokenizer, AutoModel
    import torch
    import torch.nn.functional as F

    model_name = 'bert-base-uncased'
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModel.from_pretrained(model_name)

    def get_embedding(text):
        inputs = tokenizer(text, return_tensors='pt', truncation=True, max_length=128)
        with torch.no_grad():
            outputs = model(**inputs)
        return outputs.last_hidden_state[:, 0, :].squeeze()  # [CLS] token

    s1 = get_embedding('How do I reset my password?')
    s2 = get_embedding('Steps to change account password')
    s3 = get_embedding('Best pizza recipe')

    print('s1 vs s2:', F.cosine_similarity(s1.unsqueeze(0), s2.unsqueeze(0)).item())
    print('s1 vs s3:', F.cosine_similarity(s1.unsqueeze(0), s3.unsqueeze(0)).item())
except ImportError:
    print('pip install transformers torch')

try:
    from transformers import pipeline

    qa = pipeline('question-answering', model='distilbert-base-cased-distilled-squad')

    context = '''
    Python was created by Guido van Rossum and first released in 1991.
    It was designed with an emphasis on code readability and simplicity.
    Python supports multiple programming paradigms including procedural,
    object-oriented, and functional programming.
    '''

    questions = [
        'Who created Python?',
        'When was Python first released?',
        'What paradigms does Python support?',
    ]
    for q in questions:
        answer = qa({'question': q, 'context': context})
        print(f'Q: {q}')
        print(f'A: {answer["answer"]} (score: {answer["score"]:.3f})\n')
except ImportError:
    print('pip install transformers torch')

try:
    from transformers import pipeline

    generator = pipeline('text-generation', model='gpt2', max_new_tokens=80)

    prompts = [
        'The future of artificial intelligence is',
        'Data science has transformed the way we',
    ]
    for prompt in prompts:
        result = generator(prompt, num_return_sequences=1, do_sample=True, temperature=0.7)
        print(f'Prompt: {prompt}')
        print(f'Generated: {result[0]["generated_text"]}\n')
except ImportError:
    print('pip install transformers torch')

try:
    from transformers import AutoTokenizer, AutoModelForSequenceClassification
    from transformers import TrainingArguments, Trainer
    import torch

    model_name = 'distilbert-base-uncased'
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

    # Example: tokenize a batch
    texts = ['I love this product!', 'Terrible experience.']
    labels = [1, 0]
    encodings = tokenizer(texts, truncation=True, padding=True, return_tensors='pt')

    # In real fine-tuning, wrap in a Dataset and use Trainer:
    # training_args = TrainingArguments(output_dir='./results', num_train_epochs=3)
    # trainer = Trainer(model=model, args=training_args, ...)
    # trainer.train()

    print('Model ready for fine-tuning')
    print('Tokenized input_ids shape:', encodings['input_ids'].shape)
except ImportError:
    print('pip install transformers torch datasets')

try:
    import spacy
    from collections import Counter
    nlp = spacy.load('en_core_web_sm')

    def analyze_text(text: str) -> dict:
        doc = nlp(text)
        return {
            'word_count': len([t for t in doc if not t.is_punct]),
            'sentences':  len(list(doc.sents)),
            'entities':   [(e.text, e.label_) for e in doc.ents],
            'top_nouns':  Counter(t.lemma_ for t in doc if t.pos_ == 'NOUN').most_common(3),
            'top_verbs':  Counter(t.lemma_ for t in doc if t.pos_ == 'VERB').most_common(3),
        }

    text = 'Elon Musk launched SpaceX rockets in 2020. Tesla reported record profits in Q4.'
    result = analyze_text(text)
    for k, v in result.items():
        print(f'{k}: {v}')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

try:
    import spacy
    import time
    nlp = spacy.load('en_core_web_sm')

    texts = [f'Document {i}: Apple and Google are tech giants in Silicon Valley.' for i in range(50)]

    # Sequential processing
    t0 = time.time()
    results_seq = [nlp(t) for t in texts]
    t_seq = time.time() - t0

    # Batch processing with nlp.pipe
    t0 = time.time()
    results_batch = list(nlp.pipe(texts, batch_size=16))
    t_batch = time.time() - t0

    print(f'Sequential: {t_seq:.3f}s')
    print(f'Batch pipe: {t_batch:.3f}s')
    print(f'Speedup: {t_seq/t_batch:.1f}x')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

# Run with: uvicorn app:app --reload
# pip install fastapi uvicorn

FASTAPI_APP = '''
from fastapi import FastAPI
from pydantic import BaseModel
import nltk
nltk.download("vader_lexicon", quiet=True)
from nltk.sentiment.vader import SentimentIntensityAnalyzer

app = FastAPI()
sia = SentimentIntensityAnalyzer()

class TextRequest(BaseModel):
    text: str

@app.post("/sentiment")
def analyze_sentiment(req: TextRequest):
    scores = sia.polarity_scores(req.text)
    label = "positive" if scores["compound"] > 0.05 else "negative" if scores["compound"] < -0.05 else "neutral"
    return {"label": label, "scores": scores}

@app.get("/health")
def health():
    return {"status": "ok"}
'''

print(FASTAPI_APP)

from sklearn.metrics import precision_recall_fscore_support, classification_report

# BIO tagging evaluation example
# True entity spans vs predicted entity spans
true_entities = [
    {('Apple', 'ORG'), ('Tim Cook', 'PERSON'), ('Cupertino', 'GPE')},
    {('Google', 'ORG'), ('Sundar Pichai', 'PERSON')},
]
pred_entities = [
    {('Apple', 'ORG'), ('Tim Cook', 'PERSON')},     # missed Cupertino
    {('Google', 'ORG'), ('Sundar Pichai', 'PERSON'), ('Mountain View', 'GPE')},  # extra FP
]

def ner_metrics(true_list, pred_list):
    tp = sum(len(t & p) for t, p in zip(true_list, pred_list))
    fp = sum(len(p - t) for t, p in zip(true_list, pred_list))
    fn = sum(len(t - p) for t, p in zip(true_list, pred_list))
    prec = tp / (tp + fp) if tp + fp > 0 else 0
    rec  = tp / (tp + fn) if tp + fn > 0 else 0
    f1   = 2 * prec * rec / (prec + rec) if prec + rec > 0 else 0
    return {'precision': prec, 'recall': rec, 'f1': f1}

print(ner_metrics(true_entities, pred_entities))

import re

text = '''
Elon Musk founded SpaceX in 2002. Jeff Bezos founded Amazon in 1994.
Tim Cook joined Apple in 1998 and became CEO in 2011.
Sam Altman was appointed CEO of OpenAI in 2019.
'''

# Pattern: Person + founded/joined/became + Org + in + Year
founded_pat = re.compile(
    r'([A-Z][a-z]+ [A-Z][a-z]+) (founded|joined|became \w+) ([A-Z][a-zA-Z]+) in (\d{4})'
)

relations = []
for m in founded_pat.finditer(text):
    relations.append({
        'person': m.group(1),
        'relation': m.group(2),
        'org': m.group(3),
        'year': m.group(4)
    })

for r in relations:
    print(f"{r['person']} --[{r['relation']}]--> {r['org']} ({r['year']})")

try:
    import spacy
    nlp = spacy.load('en_core_web_sm')
    text = 'Apple acquired Beats Electronics for 3 billion dollars in 2014. Google bought YouTube in 2006.'
    doc = nlp(text)
    print('Named Entities:')
    for ent in doc.ents:
        print(f'  {ent.text!r:25s} -> {ent.label_}')
    print('\nAcquisition relations (nsubj + dobj pattern):')
    for token in doc:
        if token.lemma_ in ('acquire', 'buy', 'purchase'):
            subj = [t.text for t in token.lefts  if t.dep_ in ('nsubj', 'nsubjpass')]
            obj  = [t.text for t in token.rights if t.dep_ in ('dobj', 'attr')]
            if subj and obj:
                print(f'  {subj[0]} --[{token.text}]--> {obj[0]}')
except OSError:
    print('Run: python -m spacy download en_core_web_sm')

import re
from dataclasses import dataclass, field
from typing import List

@dataclass
class JobPosting:
    title: str = ''
    company: str = ''
    location: str = ''
    salary: str = ''
    skills: List[str] = field(default_factory=list)

def extract_job_info(text: str) -> JobPosting:
    job = JobPosting()
    if m := re.search(r'(?:Title|Position|Role):\s*(.+)', text, re.I): job.title = m.group(1).strip()
    if m := re.search(r'Company:\s*(.+)', text, re.I): job.company = m.group(1).strip()
    if m := re.search(r'Location:\s*(.+)', text, re.I): job.location = m.group(1).strip()
    if m := re.search(r'Salary:\s*(\$[\d,]+ ?- ?\$[\d,]+|\$[\d,]+)', text, re.I): job.salary = m.group(1)
    skills_m = re.findall(r'\b(Python|SQL|Java|TensorFlow|PyTorch|Docker|Kubernetes|AWS|GCP)\b', text)
    job.skills = list(set(skills_m))
    return job

posting = '''
Title: Senior Data Scientist
Company: TechCorp Inc.
Location: San Francisco, CA
Salary: $150,000 - $200,000
Requirements: Python, SQL, TensorFlow, Docker, AWS experience preferred.
'''
job = extract_job_info(posting)
print(f'Title:    {job.title}')
print(f'Company:  {job.company}')
print(f'Location: {job.location}')
print(f'Salary:   {job.salary}')
print(f'Skills:   {sorted(job.skills)}')

import re
from collections import defaultdict

# Simple event extraction using trigger words
EVENT_TRIGGERS = {
    'acquisition': ['acquired', 'bought', 'purchased', 'merged with', 'took over'],
    'funding':     ['raised', 'secured', 'received funding', 'closed round'],
    'launch':      ['launched', 'released', 'unveiled', 'announced', 'introduced'],
    'partnership': ['partnered', 'collaborated', 'teamed up', 'joined forces'],
}

MONEY_RE = re.compile(r'\$[\d.,]+[BMK]?\s*(?:billion|million|thousand)?', re.I)

def extract_events(sentences):
    events = []
    for sent in sentences:
        sent_lower = sent.lower()
        for etype, triggers in EVENT_TRIGGERS.items():
            for trig in triggers:
                if trig in sent_lower:
                    money = MONEY_RE.findall(sent)
                    events.append({'type': etype, 'trigger': trig, 'money': money, 'text': sent[:80]})
                    break
    return events

news = [
    'Google acquired DeepMind for $500 million in 2014.',
    'OpenAI raised $6.6 billion in its latest funding round.',
    'Apple launched its Vision Pro headset at WWDC 2023.',
    'Meta and Microsoft partnered on enterprise AI solutions.',
]
for ev in extract_events(news):
    print(f"[{ev['type'].upper()}] trigger='{ev['trigger']}' money={ev['money']}")
    print(f"  {ev['text']}")

from collections import Counter
import math

def ngram_counts(tokens, n):
    return Counter(tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1))

def bleu_score(reference: str, hypothesis: str, max_n: int = 4) -> float:
    ref_tokens  = reference.lower().split()
    hyp_tokens  = hypothesis.lower().split()
    if not hyp_tokens:
        return 0.0
    # Brevity penalty
    bp = min(1.0, math.exp(1 - len(ref_tokens)/len(hyp_tokens)))
    scores = []
    for n in range(1, max_n + 1):
        ref_ng  = ngram_counts(ref_tokens, n)
        hyp_ng  = ngram_counts(hyp_tokens, n)
        clipped = sum(min(c, ref_ng[ng]) for ng, c in hyp_ng.items())
        total   = sum(hyp_ng.values())
        if total == 0:
            scores.append(0.0)
        else:
            scores.append(clipped / total)
    # Geometric mean of precisions
    log_avg = sum(math.log(s) if s > 0 else -999 for s in scores) / max_n
    bleu = bp * math.exp(log_avg)
    print(f'Reference:  {reference}')
    print(f'Hypothesis: {hypothesis}')
    print(f'N-gram precisions: {[round(s,3) for s in scores]}')
    print(f'BLEU-{max_n}: {bleu:.4f}')
    return bleu

bleu_score(
    'The cat sat on the mat',
    'The cat is on the mat'
)
bleu_score(
    'The cat sat on the mat',
    'A dog lay on a rug'
)

import numpy as np

# Demonstrate encoder-decoder idea without deep learning framework
np.random.seed(42)

# Toy vocabulary
vocab = {'<pad>': 0, '<sos>': 1, '<eos>': 2, 'hello': 3, 'world': 4,
         'hola': 5, 'mundo': 6, 'bonjour': 7, 'monde': 8}
idx2word = {v: k for k, v in vocab.items()}

# Random embeddings (in practice: learned)
EMB_DIM = 8
embeddings = np.random.randn(len(vocab), EMB_DIM) * 0.1

def encode(sentence: str) -> np.ndarray:
    tokens = [vocab.get(w, 0) for w in sentence.lower().split()]
    vecs = [embeddings[t] for t in tokens]
    return np.mean(vecs, axis=0)  # mean pooling = context vector

def cosine_sim(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-8)

src = encode('hello world')
print('Encoder output (context vector):', src.round(3))

# In real seq2seq: decoder generates target tokens one by one
# conditioned on context vector + previous generated token
print('\nEncoder-Decoder flow:')
print('  Source: hello world')
print('  Encoder -> context vector (shape:', src.shape, ')')
print('  Decoder: <sos> -> hola -> mundo -> <eos>')
print('  At each step: P(word | context, prev_token)')

import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

# Simulate attention weights for 'The cat sat on the mat' -> 'Le chat etait sur le tapis'
np.random.seed(42)
src_words = ['The', 'cat', 'sat', 'on', 'the', 'mat']
tgt_words = ['Le', 'chat', 'etait', 'sur', 'le', 'tapis']

# Simulate attention weights (in practice: softmax(Q @ K.T / sqrt(d_k)))
# Diagonal-dominant = good alignment
raw = np.random.rand(6, 6)
# Make it more diagonal (word alignments)
for i in range(6):
    raw[i, i] += 2.0
raw[0, 4] += 1.0  # 'the' aligns with 'le'
attention = np.exp(raw) / np.exp(raw).sum(axis=1, keepdims=True)

fig, ax = plt.subplots(figsize=(7, 5))
im = ax.imshow(attention, cmap='Blues', vmin=0, vmax=1)
ax.set_xticks(range(6)); ax.set_xticklabels(src_words, rotation=45, ha='right')
ax.set_yticks(range(6)); ax.set_yticklabels(tgt_words)
ax.set_xlabel('Source'); ax.set_ylabel('Target')
ax.set_title('Attention Weights')
plt.colorbar(im, ax=ax); plt.tight_layout()
plt.savefig('attention_weights.png', dpi=80); plt.close()
print('Saved attention_weights.png')
print('Attention row sums:', attention.sum(axis=1).round(3))

try:
    from transformers import pipeline
    # Zero-shot: use a pretrained translation model
    translator = pipeline('translation_en_to_fr', model='Helsinki-NLP/opus-mt-en-fr')
    sentences = [
        'Machine learning is transforming healthcare.',
        'The quick brown fox jumps over the lazy dog.',
        'Data science requires statistics, programming, and domain knowledge.',
    ]
    for sent in sentences:
        result = translator(sent, max_length=128)[0]['translation_text']
        print(f'EN: {sent}')
        print(f'FR: {result}\n')
except ImportError:
    print('pip install transformers sentencepiece')
    print('\nExample output:')
    print('EN: Machine learning is transforming healthcare.')
    print('FR: L apprentissage automatique transforme les soins de sante.')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

corpus = [
    'Python is a versatile programming language for data science and web development.',
    'Machine learning models require large amounts of training data.',
    'Neural networks are inspired by the structure of the human brain.',
    'Natural language processing enables computers to understand human text.',
    'Deep learning achieves state-of-the-art results on image classification tasks.',
    'Reinforcement learning trains agents through rewards and penalties.',
    'Transfer learning fine-tunes pre-trained models on new tasks.',
    'Transformers use self-attention to process sequences in parallel.',
]

vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1, 2))
tfidf_matrix = vectorizer.fit_transform(corpus)

def search(query: str, top_k: int = 3) -> list:
    q_vec = vectorizer.transform([query])
    sims  = cosine_similarity(q_vec, tfidf_matrix).flatten()
    top   = np.argsort(sims)[::-1][:top_k]
    return [(corpus[i][:70], round(sims[i], 4)) for i in top]

for q in ['how do neural networks learn', 'NLP text processing', 'image recognition']:
    print(f'Query: {q}')
    for doc, score in search(q):
        print(f'  [{score:.4f}] {doc}')
    print()

import numpy as np
from collections import Counter
import math

class BM25:
    def __init__(self, corpus, k1=1.5, b=0.75):
        self.corpus  = [doc.lower().split() for doc in corpus]
        self.k1, self.b = k1, b
        self.n = len(self.corpus)
        self.avgdl = np.mean([len(d) for d in self.corpus])
        self.df = {}
        for doc in self.corpus:
            for term in set(doc):
                self.df[term] = self.df.get(term, 0) + 1

    def score(self, query: str, doc_id: int) -> float:
        query_terms = query.lower().split()
        doc = self.corpus[doc_id]
        doc_len = len(doc)
        tf = Counter(doc)
        score = 0.0
        for term in query_terms:
            if term not in self.df: continue
            idf = math.log((self.n - self.df[term] + 0.5) / (self.df[term] + 0.5) + 1)
            freq = tf.get(term, 0)
            tf_score = freq * (self.k1 + 1) / (freq + self.k1 * (1 - self.b + self.b * doc_len / self.avgdl))
            score += idf * tf_score
        return score

    def retrieve(self, query: str, top_k: int = 3):
        scores = [(i, self.score(query, i)) for i in range(self.n)]
        return sorted(scores, key=lambda x: -x[1])[:top_k]

docs = ['Python machine learning tutorial', 'Deep neural network architectures', 'Python web scraping guide', 'Transformer models for NLP', 'Data science with Python pandas']
bm25 = BM25(docs)
print('BM25 results for "Python NLP":')
for idx, score in bm25.retrieve('Python NLP'):
    print(f'  [{score:.3f}] {docs[idx]}')

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

# Simulate sentence embeddings (in practice: use SentenceTransformer)
np.random.seed(42)

docs = [
    'How to train a neural network',
    'Python list comprehension tutorial',
    'Best practices for REST API design',
    'Introduction to gradient descent optimization',
    'SQL window functions explained',
    'Backpropagation algorithm explained',
]

# Simulate embeddings (normally: model.encode(docs))
# Make 'neural network' and 'gradient descent' semantically similar
EMB_DIM = 16
base_embs = np.random.randn(len(docs), EMB_DIM)
# Make neural network docs cluster together
for i in [0, 3, 5]:
    base_embs[i] += np.array([2]*4 + [0]*12)  # shared direction
base_embs /= np.linalg.norm(base_embs, axis=1, keepdims=True)

def dense_search(query_emb, doc_embs, top_k=3):
    sims = cosine_similarity(query_emb.reshape(1,-1), doc_embs).flatten()
    top  = np.argsort(sims)[::-1][:top_k]
    return [(docs[i], sims[i]) for i in top]

# Query embedding (similar to NN docs)
query_emb = base_embs[0] + np.random.randn(EMB_DIM) * 0.1
query_emb /= np.linalg.norm(query_emb)
print('Dense search results for [neural network query]:')
for doc, sim in dense_search(query_emb, base_embs):
    print(f'  [{sim:.3f}] {doc}')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

# Knowledge base
knowledge_base = [
    'Python was created by Guido van Rossum in 1991.',
    'NumPy provides N-dimensional array support and math functions.',
    'Pandas is built on NumPy and provides DataFrame structures for data analysis.',
    'Scikit-learn offers machine learning algorithms for classification, regression, and clustering.',
    'Matplotlib is the most popular Python plotting library.',
    'TensorFlow and PyTorch are the two leading deep learning frameworks.',
]

vect = TfidfVectorizer(stop_words='english')
kb_matrix = vect.fit_transform(knowledge_base)

def retrieve(query: str, top_k: int = 2) -> list:
    q_vec = vect.transform([query])
    sims  = cosine_similarity(q_vec, kb_matrix).flatten()
    top   = np.argsort(sims)[::-1][:top_k]
    return [knowledge_base[i] for i in top]

def rag_answer(query: str) -> str:
    """Minimal RAG: retrieve context, then format answer."""
    context = retrieve(query)
    context_str = ' '.join(context)
    # In real RAG: pass context + query to LLM (e.g. Claude/GPT)
    # Here: template-based answer simulation
    answer = f'Based on retrieved context: {context_str[:120]}...'
    return answer

queries = ['What is pandas?', 'Who created Python?', 'deep learning frameworks']
for q in queries:
    print(f'Q: {q}')
    print(f'Retrieved: {retrieve(q)[0][:60]}')
    print()

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
texts = [
    "The stock market crashed on Monday.",
    "Transformers revolutionized NLP in 2017!"
]
encoded = tokenizer(texts, padding=True, truncation=True, max_length=32, return_tensors="pt")
print("Input IDs shape:", encoded["input_ids"].shape)
for i, text in enumerate(texts):
    tokens = tokenizer.convert_ids_to_tokens(encoded["input_ids"][i])
    tokens = [t for t in tokens if t != "[PAD]"]
    print(f"\nText {i+1}: {tokens}")
print("\nVocab size:", tokenizer.vocab_size)

from transformers import pipeline
classifier = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
reviews = [
    "This product exceeded all my expectations! Absolutely fantastic.",
    "Terrible quality. Broke after one day. Very disappointed.",
    "It is okay, nothing special but gets the job done.",
]
results = classifier(reviews)
for text, result in zip(reviews, results):
    label = result["label"]
    score = result["score"]
    print(f"[{label} {score:.3f}] {text[:50]}...")

from transformers import pipeline
classifier = pipeline("zero-shot-classification",
                      model="facebook/bart-large-mnli")
text = "The Federal Reserve raised interest rates by 25 basis points today."
candidate_labels = ["finance", "sports", "technology", "politics", "health"]
result = classifier(text, candidate_labels)
print("Text:", text[:70])
print("\nClassification scores:")
for label, score in zip(result["labels"], result["scores"]):
    bar = "#" * int(score * 30)
    print(f"  {label:<12} {score:.4f}  {bar}")

import spacy
nlp = spacy.load("en_core_web_sm")
texts = [
    "Apple Inc. CEO Tim Cook announced new products at WWDC in San Francisco.",
    "On March 14, 2023, the Fed raised rates by 25bps, affecting $4.5T in bonds.",
]
for text in texts:
    doc = nlp(text)
    print(f"Text: {text[:65]}...")
    print("Entities:")
    for ent in doc.ents:
        print(f"  [{ent.label_:<10}] '{ent.text}'")
    print()

import spacy
from spacy.matcher import Matcher
nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)
# Match product codes like "SKU-12345" or "PROD-ABC99"
pattern = [{"TEXT": {"REGEX": r"(SKU|PROD|ITEM)-[A-Z0-9]{3,8}"}}]
matcher.add("PRODUCT_CODE", [pattern])
texts = [
    "Customer ordered SKU-48291 and PROD-XR99 but ITEM-ZZ001 was out of stock.",
    "Return request for SKU-11100 received from warehouse.",
]
for text in texts:
    doc = nlp(text)
    matches = matcher(doc)
    codes = [doc[start:end].text for _, start, end in matches]
    print(f"Text: {text}")
    print(f"Product codes found: {codes}\n")

import spacy
nlp = spacy.load("en_core_web_sm")
text = "Elon Musk founded SpaceX in 2002. Jeff Bezos started Amazon in 1994."
doc = nlp(text)
print("Subject-Verb-Object triples:")
for sent in doc.sents:
    for token in sent:
        if token.dep_ == "ROOT":
            subj = [c.text for c in token.children if c.dep_ in ("nsubj","nsubjpass")]
            obj  = [c.text for c in token.children if c.dep_ in ("dobj","attr","pobj")]
            if subj and obj:
                print(f"  ({subj[0]}) --[{token.text}]--> ({obj[0]})")
print("\nNamed Entities:")
for ent in doc.ents:
    print(f"  {ent.text:<15} [{ent.label_}]")

from transformers import pipeline
generator = pipeline("text-generation", model="gpt2", max_new_tokens=60)
prompts = [
    "The future of artificial intelligence is",
    "In 2035, data scientists will",
]
for prompt in prompts:
    outputs = generator(prompt, num_return_sequences=2, temperature=0.8, do_sample=True)
    print(f"Prompt: {prompt}")
    for i, out in enumerate(outputs, 1):
        generated = out["generated_text"][len(prompt):]
        print(f"  [{i}] ...{generated[:80]}")
    print()

# Demonstrates prompt engineering patterns (no API key needed — shows templates)
import json

def build_extraction_prompt(text, fields):
    field_list = ", ".join(f'"{f}"' for f in fields)
    return f"""Extract the following fields from the text below.
Return ONLY valid JSON with keys: {field_list}.
If a field is not found, use null.

Text: {text}

JSON output:"""

texts = [
    "Order #4521 placed by Sarah Johnson on 2024-03-15 for $289.99. Ships to Chicago, IL.",
    "Meeting scheduled with Dr. Patel at Boston General Hospital on Tuesday at 2pm."
]
fields_order   = ["order_id", "customer_name", "date", "amount", "city"]
fields_meeting = ["person", "organization", "day", "time"]
for text, fields in zip(texts, [fields_order, fields_meeting]):
    prompt = build_extraction_prompt(text, fields)
    print("=== Prompt Template ===")
    print(prompt[:200])
    print("...")
    print()

from transformers import pipeline
summarizer = pipeline("summarization", model="facebook/bart-large-cnn",
                      max_length=60, min_length=20, do_sample=False)
article = """
Scientists at MIT have developed a new AI system capable of predicting protein
folding structures with greater accuracy than any previous model. The breakthrough,
published in Nature, combines deep learning with molecular dynamics simulations.
Researchers tested the system on over 10,000 known protein structures and achieved
98.5% accuracy. This development could accelerate drug discovery by enabling
researchers to design proteins that target specific disease pathways. The team plans
to make the model open-source within the next six months.
"""
summary = summarizer(article.strip())[0]["summary_text"]
original_words = len(article.split())
summary_words  = len(summary.split())
print(f"Original: {original_words} words")
print(f"Summary ({summary_words} words):")
print(summary)

import spacy
from collections import Counter

# Load spaCy model (run: python -m spacy download en_core_web_sm)
try:
    nlp = spacy.load("en_core_web_sm")
except OSError:
    print("Run: python -m spacy download en_core_web_sm")
    nlp = None

text = (
    "Apple Inc. CEO Tim Cook announced in San Francisco on January 15, 2024 "
    "that the company would invest $1 billion in AI research. "
    "The partnership with OpenAI and Microsoft was confirmed by Google."
)

if nlp:
    doc = nlp(text)
    print("Named Entities:")
    for ent in doc.ents:
        print(f"  {ent.text:<30} [{ent.label_}] - {spacy.explain(ent.label_)}")

    # Count entity types
    type_counts = Counter(ent.label_ for ent in doc.ents)
    print("\nEntity type counts:", dict(type_counts))
else:
    # Simulate output structure
    entities = [
        ("Apple Inc.", "ORG", "Companies"), ("Tim Cook", "PERSON", "People"),
        ("San Francisco", "GPE", "Geo-political"), ("January 15, 2024", "DATE", "Dates"),
        ("$1 billion", "MONEY", "Monetary"), ("OpenAI", "ORG", "Companies"),
        ("Microsoft", "ORG", "Companies"), ("Google", "ORG", "Companies"),
    ]
    for text_ent, label, explain in entities:
        print(f"  {text_ent:<30} [{label}] - {explain}")

import spacy
from spacy.language import Language

try:
    nlp = spacy.load("en_core_web_sm")
except OSError:
    nlp = spacy.blank("en")

# Add EntityRuler for custom entities
ruler = nlp.add_pipe("entity_ruler", before="ner" if "ner" in nlp.pipe_names else "last")

# Define custom patterns
patterns = [
    {"label": "ML_MODEL", "pattern": "BERT"},
    {"label": "ML_MODEL", "pattern": "GPT-4"},
    {"label": "ML_MODEL", "pattern": "ResNet"},
    {"label": "ML_TASK", "pattern": "named entity recognition"},
    {"label": "ML_TASK", "pattern": "sentiment analysis"},
    {"label": "DATASET", "pattern": [{"LOWER": "imagenet"}]},
    {"label": "DATASET", "pattern": "CIFAR-10"},
]
ruler.add_patterns(patterns)

test_texts = [
    "BERT and GPT-4 are popular ML models for named entity recognition.",
    "ResNet was trained on ImageNet and CIFAR-10 datasets.",
    "sentiment analysis using BERT achieves state-of-the-art results.",
]

for text in test_texts:
    doc = nlp(text)
    ents = [(e.text, e.label_) for e in doc.ents]
    print(f"Text: {text[:50]}...")
    print(f"  Entities: {ents}\n")

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Document corpus
corpus = [
    "machine learning algorithms for classification",
    "deep learning neural networks for image recognition",
    "natural language processing text classification",
    "computer vision object detection algorithms",
    "transformer models for text generation",
    "reinforcement learning reward optimization",
]

# Build TF-IDF matrix
vectorizer = TfidfVectorizer(ngram_range=(1, 2))
tfidf_matrix = vectorizer.fit_transform(corpus)
print(f"TF-IDF matrix: {tfidf_matrix.shape}")

# Semantic search function
def search(query, top_k=3):
    query_vec = vectorizer.transform([query])
    scores = cosine_similarity(query_vec, tfidf_matrix)[0]
    top_idx = np.argsort(scores)[::-1][:top_k]
    return [(corpus[i], round(float(scores[i]), 4)) for i in top_idx]

queries = [
    "text classification with deep learning",
    "visual recognition algorithms",
]
for q in queries:
    print(f"\nQuery: '{q}'")
    for doc, score in search(q):
        print(f"  [{score:.4f}] {doc}")

import numpy as np
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Create co-occurrence matrix (simplified Word2Vec concept)
sentences = [
    "the cat sat on the mat",
    "the dog lay on the rug",
    "cats and dogs are pets",
    "machine learning models learn patterns",
    "deep learning uses neural networks",
    "neural networks learn representations",
]

# Build co-occurrence proxy via SVD on count matrix
vectorizer = CountVectorizer(min_df=1)
X = vectorizer.fit_transform(sentences)

# SVD to get dense embeddings (like word2vec)
svd = TruncatedSVD(n_components=8, random_state=42)
embeddings = svd.fit_transform(X)

print("Sentence embeddings shape:", embeddings.shape)

# Find similar sentences
def find_similar(idx, top_k=3):
    sims = cosine_similarity([embeddings[idx]], embeddings)[0]
    sims[idx] = -1  # exclude self
    top = np.argsort(sims)[::-1][:top_k]
    return [(sentences[i], round(float(sims[i]), 4)) for i in top]

for i in [0, 3]:
    print(f"\nSimilar to: '{sentences[i]}'")
    for sent, sim in find_similar(i):
        print(f"  [{sim:.4f}] {sent}")

# pip install transformers torch
# Using Hugging Face pipeline (simulated output shown)
try:
    from transformers import pipeline

    # Zero-shot classification (no fine-tuning needed)
    classifier = pipeline("zero-shot-classification",
                          model="facebook/bart-large-mnli")

    texts = [
        "The product quality is excellent and delivery was fast!",
        "Terrible service, waited 3 weeks and got wrong item.",
        "The item is okay, nothing special but works as expected.",
    ]

    candidate_labels = ["positive", "negative", "neutral"]

    for text in texts:
        result = classifier(text, candidate_labels)
        top_label = result["labels"][0]
        top_score = result["scores"][0]
        print(f"Text: {text[:50]}...")
        print(f"  -> {top_label} (score={top_score:.4f})")

except ImportError:
    # Simulate output structure
    results = [
        ("positive", 0.9823), ("negative", 0.9541), ("neutral", 0.7234)
    ]
    texts = ["Excellent product!", "Terrible service.", "Works okay."]
    for (text, (label, score)) in zip(texts, results):
        print(f"Text: {text}")
        print(f"  -> {label} (score={score:.4f})")

# Token classification and feature extraction patterns
try:
    from transformers import pipeline, AutoTokenizer, AutoModel
    import torch

    # NER pipeline
    ner = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english",
                   aggregation_strategy="simple")
    text = "Apple CEO Tim Cook announced a deal with Microsoft in New York."
    entities = ner(text)
    for ent in entities:
        print(f"  {ent['word']:<20} {ent['entity_group']:<10} score={ent['score']:.4f}")

except ImportError:
    # Demonstrate the pipeline usage pattern
    import numpy as np
    print("Simulated NER output (install transformers for real results):")
    entities = [
        {"word": "Apple", "entity_group": "ORG", "score": 0.9987},
        {"word": "Tim Cook", "entity_group": "PER", "score": 0.9945},
        {"word": "Microsoft", "entity_group": "ORG", "score": 0.9978},
        {"word": "New York", "entity_group": "LOC", "score": 0.9923},
    ]
    for ent in entities:
        print(f"  {ent['word']:<20} {ent['entity_group']:<10} score={ent['score']:.4f}")

    # Simulate sentence embeddings
    print("\nSimulating sentence embeddings (mean pooling over tokens):")
    batch_size, seq_len, hidden = 2, 128, 768
    token_embeddings = np.random.randn(batch_size, seq_len, hidden)
    sentence_embeddings = token_embeddings.mean(axis=1)
    print(f"  Input shape: {token_embeddings.shape}")
    print(f"  Sentence embedding shape: {sentence_embeddings.shape}")

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import numpy as np

# Sample corpus
documents = [
    "machine learning neural networks deep learning artificial intelligence",
    "python programming data science numpy pandas matplotlib",
    "stock market trading investment portfolio risk management",
    "climate change global warming carbon emissions renewable energy",
    "machine learning algorithms random forest gradient boosting",
    "python web development flask django rest api",
    "investment strategy hedge fund returns portfolio optimization",
    "solar wind energy renewable green sustainability climate",
    "deep learning computer vision image classification convolutional",
    "data analysis pandas visualization matplotlib seaborn statistics",
]

# Fit LDA
vectorizer = CountVectorizer(max_df=0.95, min_df=1, stop_words="english")
X = vectorizer.fit_transform(documents)
vocab = vectorizer.get_feature_names_out()

lda = LatentDirichletAllocation(n_components=3, random_state=42, max_iter=20)
lda.fit(X)

# Display top words per topic
print("Discovered Topics:")
for topic_id, topic in enumerate(lda.components_):
    top_words = [vocab[i] for i in topic.argsort()[:-8:-1]]
    print(f"  Topic {topic_id+1}: {', '.join(top_words)}")

# Document-topic distribution
doc_topics = lda.transform(X)
for i, doc in enumerate(documents[:3]):
    dominant = doc_topics[i].argmax() + 1
    print(f"\nDoc {i+1}: Topic {dominant} dominant ({doc_topics[i].max():.3f})")
    print(f"  '{doc[:50]}'")

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF

# Non-negative Matrix Factorization (often better coherence than LDA)
news_snippets = [
    "president signed new economic policy tax reform bill congress",
    "federal reserve interest rates inflation monetary policy",
    "championship game football team playoffs season victory",
    "basketball nba draft player trade contract signed",
    "covid vaccine efficacy clinical trial approval fda",
    "hospital treatment patient therapy drug clinical",
    "election campaign voter poll candidate debate",
    "tech company IPO stock shares market valuation",
    "championship trophy league season playoffs basketball",
    "interest rate hike federal bank economic growth",
]

vectorizer = TfidfVectorizer(max_df=0.95, min_df=1, stop_words="english")
tfidf = vectorizer.fit_transform(news_snippets)
vocab = vectorizer.get_feature_names_out()

# NMF topic modeling
nmf = NMF(n_components=4, random_state=42)
W = nmf.fit_transform(tfidf)  # document-topic
H = nmf.components_           # topic-word

print("NMF Topics (typically more coherent):")
topic_names = ["Politics", "Economy", "Sports", "Health"]
for i, (row, name) in enumerate(zip(H, topic_names)):
    top_words = [vocab[j] for j in row.argsort()[:-6:-1]]
    print(f"  Topic {i+1} ({name}): {', '.join(top_words)}")

# Dominant topic per document
for i, doc in enumerate(news_snippets[:4]):
    dominant = W[i].argmax()
    print(f"\nDoc: '{doc[:45]}...'")
    print(f"  Dominant topic: {topic_names[dominant]} ({W[i].max():.3f})")

import numpy as np
import re
from sklearn.feature_extraction.text import TfidfVectorizer

def extractive_summarize(text, n_sentences=3):
    # Split into sentences
    sentences = re.split(r"(?<=[.!?])\s+", text.strip())
    if len(sentences) <= n_sentences:
        return text

    # Score sentences by TF-IDF importance
    vectorizer = TfidfVectorizer(stop_words="english")
    try:
        tfidf = vectorizer.fit_transform(sentences)
        scores = np.array(tfidf.sum(axis=1)).flatten()
    except ValueError:
        return " ".join(sentences[:n_sentences])

    # Select top sentences in original order
    top_idx = sorted(np.argsort(scores)[-n_sentences:])
    return " ".join(sentences[i] for i in top_idx)

article = (
    "Machine learning is a subset of artificial intelligence that gives systems the ability "
    "to automatically learn and improve from experience. "
    "It focuses on developing computer programs that can access data and use it to learn for themselves. "
    "Deep learning is part of machine learning based on artificial neural networks. "
    "These networks have multiple layers and can learn representations of data with multiple levels of abstraction. "
    "Natural language processing enables computers to understand human language. "
    "Applications include chatbots, translation, and sentiment analysis. "
    "Computer vision allows machines to interpret and understand visual information. "
    "This includes image classification, object detection, and facial recognition. "
    "Reinforcement learning trains agents through reward and penalty signals."
)

summary = extractive_summarize(article, n_sentences=3)
original_words = len(article.split())
summary_words = len(summary.split())
print(f"Original: {original_words} words")
print(f"Summary:  {summary_words} words ({summary_words/original_words*100:.0f}% compression)")
print(f"\nSummary:\n{summary}")

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import re

def textrank_summarize(text, n_sentences=3, damping=0.85, iterations=50):
    sentences = re.split(r"(?<=[.!?])\s+", text.strip())
    if len(sentences) <= n_sentences:
        return " ".join(sentences)

    # Build similarity matrix
    vectorizer = TfidfVectorizer(stop_words="english")
    try:
        tfidf = vectorizer.fit_transform(sentences)
        sim_matrix = cosine_similarity(tfidf)
    except ValueError:
        return " ".join(sentences[:n_sentences])

    np.fill_diagonal(sim_matrix, 0)

    # Row-normalize
    row_sums = sim_matrix.sum(axis=1, keepdims=True)
    row_sums[row_sums == 0] = 1
    sim_matrix = sim_matrix / row_sums

    # Power iteration (PageRank-style)
    n = len(sentences)
    scores = np.ones(n) / n
    for _ in range(iterations):
        scores = (1 - damping) / n + damping * sim_matrix.T @ scores

    top_idx = sorted(np.argsort(scores)[-n_sentences:])
    return " ".join(sentences[i] for i in top_idx)

text = (
    "Python is a high-level programming language known for its simplicity. "
    "It supports multiple programming paradigms including procedural, object-oriented, and functional. "
    "Python is widely used in data science, machine learning, and web development. "
    "The language has a rich ecosystem of libraries like NumPy, Pandas, and TensorFlow. "
    "Its syntax is designed to be readable and concise, making it beginner-friendly. "
    "Python runs on all major platforms and has an active open-source community."
)

for n in [2, 3]:
    summary = textrank_summarize(text, n_sentences=n)
    ratio = len(summary.split()) / len(text.split()) * 100
    print(f"TextRank ({n} sentences, {ratio:.0f}% of original):")
    print(f"  {summary}\n")

import numpy as np
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

class ExtractiveQA:
    def __init__(self, passage_size=2):
        self.passage_size = passage_size
        self.vectorizer = TfidfVectorizer(ngram_range=(1, 2), stop_words="english")
        self.passages = []
        self.passage_matrix = None

    def index(self, text):
        # Split into overlapping passages
        sentences = re.split(r"(?<=[.!?])\s+", text.strip())
        passages = []
        for i in range(0, len(sentences), self.passage_size):
            chunk = " ".join(sentences[i:i+self.passage_size])
            if chunk.strip():
                passages.append(chunk)
        self.passages = passages
        self.passage_matrix = self.vectorizer.fit_transform(passages)

    def answer(self, question, top_k=1):
        q_vec = self.vectorizer.transform([question])
        scores = cosine_similarity(q_vec, self.passage_matrix)[0]
        top_idx = np.argsort(scores)[::-1][:top_k]
        return [(self.passages[i], round(float(scores[i]), 4)) for i in top_idx]

context = (
    "Python was created by Guido van Rossum in 1991. "
    "The language emphasizes code readability and simplicity. "
    "Python 3.0 was released in 2008 with major changes from Python 2. "
    "NumPy was created by Travis Oliphant in 2005. "
    "Pandas was developed by Wes McKinney in 2008 for data manipulation. "
    "Scikit-learn was released in 2007 and provides machine learning tools. "
    "TensorFlow was developed by Google Brain team and released in 2015. "
    "PyTorch was released by Facebook AI Research in 2016."
)

qa = ExtractiveQA(passage_size=2)
qa.index(context)

questions = [
    "When was Python created?",
    "Who created pandas?",
    "When was TensorFlow released?",
]
for q in questions:
    answer, score = qa.answer(q)[0]
    print(f"Q: {q}")
    print(f"A: {answer} [score={score}]\n")

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import re

# Simulated knowledge base (RAG knowledge store)
KB = {
    "python_history": "Python was created by Guido van Rossum, first released in 1991. Python 3 was released in 2008.",
    "ml_libraries": "Scikit-learn (2007), TensorFlow (2015), PyTorch (2016) are major ML libraries.",
    "numpy_info": "NumPy provides N-dimensional array objects. It was created by Travis Oliphant in 2005.",
    "pandas_info": "Pandas was created by Wes McKinney in 2008. It provides DataFrames for data manipulation.",
    "deep_learning": "Deep learning uses neural networks with many layers. CNNs are used for images, RNNs for sequences.",
    "transformers": "Transformers use attention mechanisms. BERT (2018) and GPT (2018) are key transformer models.",
}

# Build retrieval index
vectorizer = TfidfVectorizer(stop_words="english")
docs = list(KB.values())
keys = list(KB.keys())
index = vectorizer.fit_transform(docs)

def rag_answer(question, top_k=2):
    q_vec = vectorizer.transform([question])
    scores = cosine_similarity(q_vec, index)[0]
    top_idx = np.argsort(scores)[::-1][:top_k]

    context = " ".join(docs[i] for i in top_idx)
    retrieved_keys = [keys[i] for i in top_idx]

    # Simple extraction: find sentence most similar to question
    sents = re.split(r"(?<=[.])\s+", context)
    if not sents:
        return "No answer found."

    sent_vecs = vectorizer.transform(sents)
    sent_scores = cosine_similarity(q_vec, sent_vecs)[0]
    best_sent = sents[sent_scores.argmax()]

    return {"answer": best_sent, "sources": retrieved_keys, "context": context[:100]}

questions = ["Who created NumPy?", "What is deep learning?", "When was BERT released?"]
for q in questions:
    result = rag_answer(q)
    print(f"Q: {q}")
    print(f"  Answer: {result['answer']}")
    print(f"  Sources: {result['sources']}\n")

import random
from collections import defaultdict, Counter
import re

class NgramLM:
    def __init__(self, n=2):
        self.n = n
        self.ngrams = defaultdict(Counter)

    def train(self, texts):
        for text in texts:
            tokens = text.lower().split()
            tokens = ["<s>"] * (self.n-1) + tokens + ["</s>"]
            for i in range(len(tokens) - self.n + 1):
                context = tuple(tokens[i:i+self.n-1])
                next_token = tokens[i+self.n-1]
                self.ngrams[context][next_token] += 1

    def generate(self, max_len=20, seed=None):
        if seed:
            random.seed(seed)
        context = ("<s>",) * (self.n-1)
        tokens = []
        for _ in range(max_len):
            if context not in self.ngrams:
                break
            candidates = list(self.ngrams[context].items())
            next_tok = random.choices(
                [w for w, _ in candidates],
                weights=[c for _, c in candidates]
            )[0]
            if next_tok == "</s>":
                break
            tokens.append(next_tok)
            context = context[1:] + (next_tok,)
        return " ".join(tokens)

corpus = [
    "the cat sat on the mat and the dog sat on the rug",
    "machine learning models learn from data and improve over time",
    "deep learning uses neural networks with many layers",
    "natural language processing handles text and speech data",
    "data science involves statistics machine learning and programming",
]

model = NgramLM(n=3)
model.train(corpus)
print("Generated text (trigram LM):")
for i in range(4):
    print(f"  {i+1}: {model.generate(max_len=12, seed=i)}")

import random
from collections import defaultdict
import re

class MarkovChain:
    def __init__(self, order=2):
        self.order = order
        self.chain = defaultdict(list)
        self.starts = []

    def train(self, text):
        words = re.findall(r"\w+[.,!?]?", text.lower())
        if len(words) < self.order + 1:
            return
        self.starts.append(tuple(words[:self.order]))
        for i in range(len(words) - self.order):
            key = tuple(words[i:i+self.order])
            self.chain[key].append(words[i+self.order])

    def generate(self, n_words=30, seed=42):
        random.seed(seed)
        if not self.starts:
            return ""
        state = random.choice(self.starts)
        result = list(state)
        for _ in range(n_words - self.order):
            if state not in self.chain:
                break
            next_word = random.choice(self.chain[state])
            result.append(next_word)
            state = tuple(result[-self.order:])
        return " ".join(result).capitalize()

mc = MarkovChain(order=2)
training_data = [
    "Data science combines statistics, machine learning, and domain expertise to extract insights from data.",
    "Machine learning models learn patterns from training data and generalize to new examples.",
    "Deep learning architectures with many layers can learn hierarchical representations.",
    "Natural language processing techniques enable machines to understand and generate human language.",
]
for text in training_data:
    mc.train(text)

print("Markov Chain Generated Text:")
for seed in [1, 2, 3]:
    print(f"  Seed {seed}: {mc.generate(n_words=20, seed=seed)}")

import re
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import classification_report
from sklearn.base import BaseEstimator, TransformerMixin

class TextPreprocessor(BaseEstimator, TransformerMixin):
    def __init__(self, lowercase=True, remove_punct=True, remove_numbers=False):
        self.lowercase = lowercase
        self.remove_punct = remove_punct
        self.remove_numbers = remove_numbers

    def preprocess(self, text):
        if self.lowercase:
            text = text.lower()
        if self.remove_punct:
            text = re.sub(r"[^\w\s]", " ", text)
        if self.remove_numbers:
            text = re.sub(r"\d+", "", text)
        text = re.sub(r"\s+", " ", text).strip()
        return text

    def fit(self, X, y=None): return self
    def transform(self, X): return [self.preprocess(t) for t in X]

# Sample dataset
texts = [
    "Great product, fast shipping, very satisfied!",
    "Terrible quality, broke after one week.",
    "Average item, nothing special.",
    "Excellent! Exceeded all expectations!",
    "Disappointed with the purchase.",
    "Does the job, no complaints.",
    "Best purchase I have made this year!",
    "Waste of money, poor customer service.",
]
labels = ["pos", "neg", "neu", "pos", "neg", "neu", "pos", "neg"]

X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=0.25, random_state=42)

pipeline = Pipeline([
    ("preprocessor", TextPreprocessor(lowercase=True, remove_punct=True)),
    ("tfidf", TfidfVectorizer(ngram_range=(1, 2), sublinear_tf=True)),
    ("classifier", LogisticRegression(C=1.0, max_iter=1000, random_state=42)),
])

pipeline.fit(X_train, y_train)
print("Test accuracy:", round(pipeline.score(X_test, y_test), 4))
print(classification_report(y_test, pipeline.predict(X_test), zero_division=0))

# New predictions
new_texts = ["Amazing value!", "Completely broken on arrival.", "It works."]
for text, pred in zip(new_texts, pipeline.predict(new_texts)):
    print(f"  [{pred}] {text}")

import re
import numpy as np
from collections import deque
from datetime import datetime

class NLPProductionSystem:
    def __init__(self, model, vectorizer):
        self.model = model
        self.vectorizer = vectorizer
        self.request_log = deque(maxlen=1000)
        self.prediction_counts = {}
        self.error_rate = 0.0

    def validate_input(self, text):
        if not isinstance(text, str):
            raise ValueError("Input must be a string")
        if len(text.strip()) < 3:
            raise ValueError("Input too short (minimum 3 chars)")
        if len(text) > 10000:
            raise ValueError("Input too long (maximum 10000 chars)")
        # Check for injection-like patterns
        if re.search(r"[<>{}|\\]", text):
            text = re.sub(r"[<>{}|\\]", " ", text)
        return text.strip()

    def predict(self, text):
        ts = datetime.now().isoformat()
        try:
            clean_text = self.validate_input(text)
            # Simulate prediction
            features = self.vectorizer.transform([clean_text])
            pred = self.model.predict(features)[0]
            prob = self.model.predict_proba(features).max()

            # Log request
            log_entry = {"ts": ts, "text_len": len(clean_text), "pred": pred, "prob": round(float(prob), 4)}
            self.request_log.append(log_entry)

            # Track prediction distribution
            self.prediction_counts[pred] = self.prediction_counts.get(pred, 0) + 1

            return {"prediction": pred, "confidence": round(float(prob), 4), "status": "ok"}

        except Exception as e:
            self.error_rate = (self.error_rate * len(self.request_log) + 1) / (len(self.request_log) + 1)
            return {"error": str(e), "status": "error"}

    def health_report(self):
        total = len(self.request_log)
        return {
            "total_requests": total,
            "prediction_distribution": self.prediction_counts,
            "error_rate": round(self.error_rate, 4),
            "avg_text_length": round(np.mean([r["text_len"] for r in self.request_log]) if self.request_log else 0, 1)
        }

# Setup
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression

vec = TfidfVectorizer(ngram_range=(1,2))
texts = ["great product", "terrible quality", "okay item", "excellent service",
         "bad experience", "good value", "poor quality", "fantastic result"]
labels = ["pos","neg","neu","pos","neg","pos","neg","pos"]
vec.fit(texts)
model = LogisticRegression(max_iter=1000, random_state=42)
model.fit(vec.transform(texts), labels)

system = NLPProductionSystem(model, vec)

test_inputs = ["Amazing product!", "", "a", "Works great!", "Very poor quality...", "It is fine I guess"]
for text in test_inputs:
    result = system.predict(text)
    print(f"  Input: {repr(text):<40} -> {result}")

print("\nHealth Report:", system.health_report())