48. Manual Sample Enrichment WorkflowPremium

Title: "Gut microbiome in inflammatory bowel disease patients undergoing dietary intervention"
→ disease: "ibd"

Title: "Fecal microbiota profiles of Parkinson's Disease patients"
→ disease: "parkinsons"

Title: "Colorectal cancer tissue microbiome analysis"
→ disease: "crc"

# Step 1: Read publication metadata
pub_metadata = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_metadata",
    workspace="metadata",
    level="metadata"
)

# Step 2: Extract disease from title (in metadata JSON)
# metadata["data"] contains title, abstract, full content

# Step 3: Propagate to samples
execute_custom_code('''
import json
# Load samples
samples = pub_queue_doi_X_Y_Z_samples["samples"]

# Extract disease from publication title
pub_title = pub_queue_doi_X_Y_Z_metadata["data"].get("content", "").split("\\n")[0]

# Infer disease (simple keyword matching)
disease_map = {
    "inflammatory bowel disease": "ibd",
    "crohn": "cd",
    "colitis": "uc",
    "colorectal cancer": "crc",
    "parkinson": "parkinsons",
}

inferred_disease = None
for pattern, disease_code in disease_map.items():
    if pattern in pub_title.lower():
        inferred_disease = disease_code
        break

# Propagate to samples
if inferred_disease:
    for s in samples:
        if not s.get("disease"):
            s["disease"] = inferred_disease
            s["disease_source"] = "inferred_from_publication_title"

result = {"samples": samples, "output_key": "enriched_samples"}
''')

Methods: "We recruited 45 patients (23 males, 22 females) aged 35-65 years (mean 52 ± 8.3)
diagnosed with Crohn's disease..."

Extract:
- age_range: [35, 65]
- age_mean: 52
- sex_distribution: {"male": 0.51, "female": 0.49}
- disease: "cd"

# Step 1: Read methods section
methods = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_methods",
    workspace="metadata",
    level="metadata"
)

# Step 2: Extract demographics with regex
execute_custom_code('''
import re

methods_text = pub_queue_doi_X_Y_Z_methods["data"]["methods_text"]

# Extract age range
age_match = re.search(r'aged? (\\d+)-?(\\d+)?', methods_text, re.I)
if age_match:
    age_min = int(age_match.group(1))
    age_max = int(age_match.group(2)) if age_match.group(2) else None
    age_midpoint = (age_min + (age_max or age_min)) // 2

# Extract sex distribution
sex_match = re.search(r'(\\d+) males?, (\\d+) females?', methods_text, re.I)
if sex_match:
    males = int(sex_match.group(1))
    females = int(sex_match.group(2))
    sex_ratio = males / (males + females)

# Propagate to samples (stochastic assignment)
samples = pub_queue_doi_X_Y_Z_samples["samples"]
for i, s in enumerate(samples):
    if not s.get("age") and age_midpoint:
        s["age"] = age_midpoint
        s["age_source"] = "inferred_from_methods_section"
    if not s.get("sex") and sex_ratio:
        # Stochastic assignment maintaining distribution
        s["sex"] = "male" if (i / len(samples)) < sex_ratio else "female"
        s["sex_source"] = "inferred_from_cohort_distribution"

result = {"samples": samples}
''')

Results: "Fecal samples were collected from 89 patients, while colonic biopsies
were obtained from a subset of 34 patients during colonoscopy..."

Extract:
- sample_type distribution: fecal (89), tissue (34)
- tissue: colon (for tissue samples)

# Read full publication content
full_content = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_metadata",
    workspace="metadata",
    level="metadata"
)

# Search for tissue-specific mentions
execute_custom_code('''
import re

full_text = pub_queue_doi_X_Y_Z_metadata["data"]["content"]
samples = pub_queue_doi_X_Y_Z_samples["samples"]

# Extract tissue mentions from Results section
results_section = re.search(r'## Results(.*?)(?:## |$)', full_text, re.DOTALL)
if results_section:
    results_text = results_section.group(1)

    # Look for tissue type mentions
    if "colonic biopsy" in results_text.lower() or "colon biopsy" in results_text.lower():
        tissue_type = "colon"
    elif "ileal biopsy" in results_text.lower():
        tissue_type = "ileum"
    elif "rectal biopsy" in results_text.lower():
        tissue_type = "rectum"
    else:
        tissue_type = None

    # Propagate to samples with "tissue" or "biopsy" in isolation_source
    if tissue_type:
        for s in samples:
            iso_source = s.get("isolation_source", "").lower()
            if "tissue" in iso_source or "biopsy" in iso_source:
                if not s.get("tissue"):
                    s["tissue"] = tissue_type
                    s["tissue_source"] = "inferred_from_results_section"

result = {"samples": samples}
''')

# List publication queue entries
lobster query "List publication queue entries with handoff_ready status"

# Check metadata completeness for target entry
lobster query "Show metadata summary for entry pub_queue_doi_10_1080_19490976_2022_2046244"

# Option A: Read methods section only (faster)
lobster query "Get methods section for pub_queue_doi_10_1080_19490976_2022_2046244_methods"

# Option B: Read full publication text (comprehensive)
lobster query "Get full metadata for pub_queue_doi_10_1080_19490976_2022_2046244_metadata"

lobster query "Execute custom enrichment code for PRJNA642308 IBD study to add disease field"

# metadata_assistant will use execute_custom_code with:
# 1. Load samples from workspace
# 2. Extract disease from publication title ("inflammatory bowel disease")
# 3. Propagate to all samples: s["disease"] = "ibd", s["disease_source"] = "publication_title"
# 4. Store enriched samples in workspace

# Re-check metadata completeness
lobster query "Show enriched samples completeness for PRJNA642308"

# Expected: disease 0% → 100%

lobster query "Export enriched PRJNA642308 samples to CSV with publication context"

# Output: workspace/metadata/exports/prjna642308_enriched_YYYY-MM-DD.csv

pub_queue_doi_X_Y_Z/
├── _metadata.json             ← FULL TEXT (level="metadata")
│   ├── content: "## Introduction\n..."    ← ALL SECTIONS
│   ├── title: "Study title"
│   ├── abstract: null (embedded in content)
│   ├── authors: [...]
│   └── journal: "..."
│
├── _methods.json              ← METHODS ONLY (level="methods")
│   ├── methods_text: "## Methods\n..."
│   ├── methods_dict:
│   │   ├── software_used: [...]
│   │   ├── parameters: {...}
│   │   └── statistical_methods: [...]
│
└── _identifiers.json          ← DATASET IDS (level="metadata")
    └── identifiers: {"geo": [...], "sra": [...]}

# 1. Title only (from summary)
result = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_metadata",
    level="summary"
)
# Returns: "Title: ..., Authors: ..., Year: ..."

# 2. Methods section only
result = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_methods",
    level="methods"
)
# Returns: "## Methods\n\nThis survey study was developed..."

# 3. FULL TEXT (Introduction + Methods + Results + Discussion)
result = get_content_from_workspace(
    identifier="pub_queue_doi_X_Y_Z_metadata",
    level="metadata"
)
# Returns: JSON with "content" field containing complete publication text

# 4. Parse full metadata JSON to extract specific sections
execute_custom_code('''
import re
full_content = pub_queue_doi_X_Y_Z_metadata["data"]["content"]

# Extract Introduction
intro = re.search(r'## Introduction(.*?)(?:## |$)', full_content, re.DOTALL)
intro_text = intro.group(1) if intro else None

# Extract Results
results = re.search(r'## Results(.*?)(?:## |$)', full_content, re.DOTALL)
results_text = results.group(1) if results else None

# Extract disease mentions across ALL sections
disease_mentions = re.findall(r'(inflammatory bowel disease|crohn|colitis|parkinson)',
                              full_content, re.I)

result = {
    "intro_text": intro_text,
    "results_text": results_text,
    "disease_mentions": disease_mentions
}
''')

# Read full metadata
pub_data = get_content_from_workspace("pub_queue_X_metadata", level="metadata")

# Pattern 1: Title mentions disease explicitly
title = pub_data["data"]["content"].split("\n")[0]
if "inflammatory bowel disease" in title.lower() or "ibd" in title.lower():
    disease = "ibd"
elif "crohn" in title.lower():
    disease = "cd"
elif "colitis" in title.lower():
    disease = "uc"
elif "colorectal cancer" in title.lower() or "crc" in title.lower():
    disease = "crc"
# ... add more patterns

# Propagate to all samples
for sample in samples:
    if not sample.get("disease"):
        sample["disease"] = disease
        sample["disease_source"] = "inferred_from_publication_title"

# Read methods section
methods = get_content_from_workspace("pub_queue_X_methods", level="methods")
methods_text = methods["data"]["methods_text"]

# Extract age range
import re
age_pattern = r'aged? (\d+)[-–to ]+(\d+)?\s*years?'
age_match = re.search(age_pattern, methods_text, re.I)

if age_match:
    age_min = int(age_match.group(1))
    age_max = int(age_match.group(2)) if age_match.group(2) else age_min
    age_midpoint = (age_min + age_max) // 2

    # Propagate
    for sample in samples:
        if not sample.get("age"):
            sample["age"] = age_midpoint
            sample["age_source"] = f"inferred_from_methods (range: {age_min}-{age_max})"

# Extract sex distribution
sex_pattern = r'(\d+)\s+males?,?\s+(\d+)\s+females?'
sex_match = re.search(sex_pattern, methods_text, re.I)

if sex_match:
    males = int(sex_match.group(1))
    females = int(sex_match.group(2))
    total = males + females
    male_ratio = males / total

    # Stochastic assignment (maintains distribution)
    for i, sample in enumerate(samples):
        if not sample.get("sex"):
            sample["sex"] = "male" if (i / len(samples)) < male_ratio else "female"
            sample["sex_source"] = f"stochastic_assignment (cohort: {males}M/{females}F)"

# Read full metadata for Results section
full_content = get_content_from_workspace("pub_queue_X_metadata", level="metadata")["data"]["content"]

# Extract Results section
results_match = re.search(r'## Results(.*?)(?:## |$)', full_content, re.DOTALL)
if results_match:
    results_text = results_match.group(1)

    # Look for tissue type mentions
    tissue_patterns = {
        r'colonic? biops(?:y|ies)': 'colon',
        r'ileal biops(?:y|ies)': 'ileum',
        r'rectal biops(?:y|ies)': 'rectum',
        r'duodenal biops(?:y|ies)': 'duodenum',
    }

    tissue_inferred = {}
    for pattern, tissue_type in tissue_patterns.items():
        if re.search(pattern, results_text, re.I):
            tissue_inferred[tissue_type] = True

    # Propagate to samples with "biopsy" in isolation_source
    for sample in samples:
        iso_source = sample.get("isolation_source", "").lower()
        if "biopsy" in iso_source or "tissue" in iso_source:
            if not sample.get("tissue") and len(tissue_inferred) == 1:
                sample["tissue"] = list(tissue_inferred.keys())[0]
                sample["tissue_source"] = "inferred_from_results_section"

# Step 1: Verify current completeness
lobster query "Check metadata completeness for PRJNA642308 entry"
# Result: organism 100%, host 100%, tissue 100%, age 100%, sex 100%, disease 0%

# Step 2: Read publication metadata
lobster query "Get full metadata for pub_queue_doi_10_1080_19490976_2022_2046244_metadata"
# Result: Title contains "inflammatory bowel disease patients"

# Step 3: Enrich with disease
lobster query "Execute custom code to enrich PRJNA642308 with IBD disease from publication title"

# metadata_assistant executes:
execute_custom_code('''
samples = pub_queue_doi_10_1080_19490976_2022_2046244_samples["samples"]
for s in samples:
    s["disease"] = "ibd"
    s["disease_source"] = "inferred_from_publication_title"
result = {"samples": samples, "output_key": "prjna642308_enriched"}
''')

# Step 4: Export enriched
lobster query "Export prjna642308_enriched to CSV"
# Result: disease coverage 0% → 100%

# Good: Traceable
sample["disease"] = "ibd"
sample["disease_source"] = "inferred_from_publication_title_doi_10_1080_19490976_2022_2046244"

# Bad: Not traceable
sample["disease"] = "ibd"

# Good: Conservative estimate
age_range = "45-65 years"
sample["age"] = 55  # Midpoint
sample["age_source"] = "inferred (range: 45-65 from methods)"

# Bad: Arbitrary assignment
sample["age"] = 45  # Why not 65?

# Good: Maintains cohort distribution
# Cohort: 23 males (51%), 22 females (49%)
for i, sample in enumerate(samples):
    sample["sex"] = "male" if (i / len(samples)) < 0.51 else "female"
    sample["sex_source"] = "stochastic (cohort: 23M/22F, 51% male)"

# Bad: All male
for sample in samples:
    sample["sex"] = "male"  # Incorrect distribution

# Before enrichment
completeness_before = sum([s.get("disease") is not None for s in samples]) / len(samples)

# After enrichment
# ... enrich samples ...

# After enrichment
completeness_after = sum([s.get("disease") is not None for s in samples]) / len(samples)

print(f"Disease completeness: {completeness_before*100:.1f}% → {completeness_after*100:.1f}%")
# Only proceed if improvement > 10%

Study SRP001 (Broad Institute, 2020, MiSeq V4)
  ├── Cited by Paper A (Nature 2021) - "Diet affects microbiome"
  └── Cited by Paper B (Cell 2022) - "Meta-analysis of 100 studies"

For batch correction:
  ✓ Group by: SRP001 (same technical batch)
  ✗ Group by: Paper A vs Paper B (different citation contexts, NOT technical batches)

library(sva)
library(phyloseq)

# Read exported CSV
metadata <- read.csv("harmonized_samples.csv")

# ComBat-seq correction (batch = study)
batch <- factor(metadata$study_accession)
condition <- factor(metadata$disease)

# Adjust count matrix
adjusted_counts <- ComBat_seq(
  counts = count_matrix,
  batch = batch,
  group = condition
)

import pandas as pd
from combat.pycombat import pycombat

# Read exported CSV
metadata = pd.read_csv("harmonized_samples.csv")

# ComBat correction
adjusted_data = pycombat(
    data=count_matrix,
    batch=metadata['study_accession'].values,  # Use study, not source_*
    mod=pd.get_dummies(metadata['disease'], drop_first=True)
)

library(vegan)

# Test if study explains microbiome variation
permanova_result <- adonis2(
  distance_matrix ~ disease + study_accession,
  data = metadata,
  permutations = 999
)

# If study_accession is significant → batch effects present
# Apply correction before disease-based analysis