{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "76129cb7-363c-4efc-a10b-daf4edc7033a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "\n",
    "def read_filter_gff_data(config):\n",
    "    gff_path = os.path.join(config['input_resource']['root'], config['input_resource']['sequence'], config['preprocessing_data_config']['gff_file'])\n",
    "    print(f'Reading gff file from {gff_path}')\n",
    "    gff_df = read_genes(gff_path)\n",
    "\n",
    "    # Remove genes on chrX, chrY, and chrM\n",
    "    gff_df = gff_df[~gff_df['seqname'].isin(['chrX', 'chrY', 'chrM'])]\n",
    "\n",
    "    # Keep a few transcript_types\n",
    "    transcript_types = ['protein_coding', 'lincRNA', 'miRNA', 'snRNA', 'snoRNA', 'rRNA']\n",
    "    gff_df = gff_df[gff_df['transcript_type'].isin(transcript_types)]\n",
    "\n",
    "    na_start = gff_df['start'].isna()\n",
    "    na_end = gff_df['end'].isna()\n",
    "    na_gene = gff_df[na_start | na_end]\n",
    "    if len(na_gene) > 0:\n",
    "        print(f'Warning: {len(na_gene)} genes with NA start or end')\n",
    "        print(na_gene['gene_name'].to_string(index=False))\n",
    "\n",
    "    gff_df = gff_df[~(na_start | na_end)]\n",
    "\n",
    "    # Add TSS column\n",
    "    gff_df['tss'] = np.where(gff_df['strand'] == '+', gff_df['start'], gff_df['end']).astype(int)\n",
    "\n",
    "    tss_radius = config['loci_source_config']['atac_seq_specific_config']['promoter']['tss_radius']\n",
    "    gff_df = add_inference_loci(gff_df, tss_radius, config)\n",
    "    return gff_df\n",
    "\n",
    "def save_gene_list(gene_type_dict, config):\n",
    "    save_root = config[\"preprocessing_data_config\"][\"output\"][\"path\"] + '/gene_list'\n",
    "    os.makedirs(save_root, exist_ok=True)\n",
    "    # Save gene list and merged bed file for inference\n",
    "    for gene_list_type, gene_list_df in gene_type_dict.items():\n",
    "        # Save gene list\n",
    "        gene_list_df.to_csv(f'{save_root}/{gene_list_type}_gene_list.csv', index=False)\n",
    "        # Save bed file\n",
    "        bed_df = gene_list_df[['seqname', 'tss_inference_start', 'tss_inference_end', 'transcript_name']].copy()\n",
    "        bed_df['tss_inference_start'] = bed_df['tss_inference_start'].astype(int)\n",
    "        bed_df['tss_inference_end'] = bed_df['tss_inference_end'].astype(int)\n",
    "        bed_df['score'] = 0\n",
    "        bed_df['strand'] = gene_list_df['strand']\n",
    "        bed_df.to_csv(f'{save_root}/{gene_list_type}_gene_list.bed', sep='\\t', index=False, header=False)\n",
    "\n",
    "    # Merge loci for inference\n",
    "    merge_loci(gene_type_dict, save_root)\n",
    "\n",
    "def merge_loci(gene_type_dict, save_root):\n",
    "    # Merge overlapping loci from all gene lists\n",
    "    merged_df = pd.concat(gene_type_dict.values())\n",
    "    \n",
    "    # Sort by chromosome and start position\n",
    "    merged_df = merged_df.sort_values(['seqname', 'tss_inference_start'])\n",
    "    \n",
    "    # Merge overlapping regions\n",
    "    merged_regions = []\n",
    "    current_chrom = None\n",
    "    current_start = None \n",
    "    current_end = None\n",
    "    current_names = []\n",
    "    current_strands = []\n",
    "    \n",
    "    for _, row in merged_df.iterrows():\n",
    "        if current_chrom != row['seqname'] or row['tss_inference_start'] > current_end:\n",
    "            # Save previous merged region\n",
    "            if current_chrom is not None:\n",
    "                merged_regions.append({\n",
    "                    'seqname': current_chrom,\n",
    "                    'start': current_start,\n",
    "                    'end': current_end,\n",
    "                    'transcript_name': '|'.join(current_names),\n",
    "                    'score': 0,\n",
    "                    'strand': '|'.join(current_strands)\n",
    "                })\n",
    "            # Start new region\n",
    "            current_chrom = row['seqname']\n",
    "            current_start = row['tss_inference_start']\n",
    "            current_end = row['tss_inference_end']\n",
    "            current_names = [row['transcript_name']]\n",
    "            current_strands = [row['strand']]\n",
    "        else:\n",
    "            # Extend current region\n",
    "            current_end = max(current_end, row['tss_inference_end'])\n",
    "            current_names.append(row['transcript_name'])\n",
    "            current_strands.append(row['strand'])\n",
    "    \n",
    "    # Add final region\n",
    "    if current_chrom is not None:\n",
    "        merged_regions.append({\n",
    "            'seqname': current_chrom,\n",
    "            'start': current_start,\n",
    "            'end': current_end,\n",
    "            'transcript_name': '|'.join(current_names),\n",
    "            'score': 0,\n",
    "            'strand': '|'.join(current_strands)\n",
    "        })\n",
    "    \n",
    "    # Convert to dataframe and save\n",
    "    merged_df = pd.DataFrame(merged_regions)\n",
    "    merged_df.to_csv(f'{save_root}/merged_loci.csv', index=False)\n",
    "    merged_df.to_csv(f'{save_root}/merged_loci.bed', sep='\\t', index=False, header=False)\n",
    "\n",
    "def get_filter_location(diff_df, top_n):\n",
    "    # Make sure not too many loci and from the same gene\n",
    "    g_idx = 0\n",
    "    o_idx = 0\n",
    "    gene_set = set()\n",
    "    if 'gene_name' in diff_df.columns:\n",
    "        gene_list = diff_df['gene_name'].values\n",
    "        for g_idx, gene in enumerate(gene_list):\n",
    "            gene_set.add(gene)\n",
    "            if len(gene_set) == top_n:\n",
    "                break\n",
    "    # Make sure not too many loci and from the same region\n",
    "    overlap_threshold = 0.5\n",
    "    region_df = diff_df[['seqname', 'tss_inference_start', 'tss_inference_end']]\n",
    "    unique_region_length = 0\n",
    "    for o_idx, row in region_df.iterrows():\n",
    "        if o_idx == 0:\n",
    "            continue\n",
    "        prior_df = region_df.iloc[:o_idx]\n",
    "        same_chrom_df = prior_df[prior_df['seqname'] == row['seqname']]\n",
    "        # Calculate if there are overlaps over 50%\n",
    "        starts = same_chrom_df['tss_inference_start'].values\n",
    "        ends = same_chrom_df['tss_inference_end'].values\n",
    "        current_start = row['tss_inference_start']\n",
    "        current_end = row['tss_inference_end']\n",
    "        overlaps = np.maximum(0, np.minimum(current_end, ends) - np.maximum(current_start, starts))\n",
    "        half_length = (current_end - current_start) / 2\n",
    "        if np.sum(overlaps) / half_length > overlap_threshold:\n",
    "            continue\n",
    "        unique_region_length += 1\n",
    "        if unique_region_length > top_n:\n",
    "            break\n",
    "    return max(g_idx, o_idx), max(len(gene_set), unique_region_length)\n",
    "\n",
    "def filter_by_rna_promoter_accessibility(gene_list_dict, config):\n",
    "\n",
    "    for gene_list_type, diff_df in gene_list_dict.items():\n",
    "        # Remove genes with small accessibility \n",
    "        accessibility_threshold = config['loci_source_config']['atac']['active_locus_threshold']\n",
    "        cross_condition_max_accessibility = diff_df[['experimental_max_accessibility', 'control_max_accessibility']].max(axis=1)\n",
    "        select_idx = cross_condition_max_accessibility > accessibility_threshold\n",
    "        gene_list_dict[gene_list_type] = diff_df[select_idx]\n",
    "\n",
    "    upregulated_df, downregulated_df, conserved_df = gene_list_dict['upregulated'], gene_list_dict['downregulated'], gene_list_dict['conserved']\n",
    "\n",
    "    acc_log2fc_threshold = config['loci_source_config']['atac']['log2fc_threshold']\n",
    "\n",
    "    num_diff = config['loci_source_config']['rna']['top_diff_genes']\n",
    "    upregulated_df = add_log2fc(upregulated_df)\n",
    "    upregulated_df = upregulated_df[upregulated_df['accessibility_log2fc'] > acc_log2fc_threshold]\n",
    "    upregulated_df = upregulated_df.sort_values(by='abs_rna_acc_log2fc_sum', ascending=False).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(upregulated_df, num_diff) # Get the location of the top num_diff unique genes\n",
    "    upregulated_df = upregulated_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_diff:\n",
    "        print(f'Warning: Only {num_unique} unique upregulated genes found after filtering by promoter accessibility')\n",
    "\n",
    "    downregulated_df = add_log2fc(downregulated_df)\n",
    "    downregulated_df = downregulated_df[downregulated_df['accessibility_log2fc'] < -acc_log2fc_threshold]\n",
    "    downregulated_df = downregulated_df.sort_values(by='abs_rna_acc_log2fc_sum', ascending=False).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(downregulated_df, num_diff) # Get the location of the top num_diff unique genes\n",
    "    downregulated_df = downregulated_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_diff:\n",
    "        print(f'Warning: Only {num_unique} unique downregulated genes found after filtering by promoter accessibility')\n",
    "\n",
    "    num_conserved = config['loci_source_config']['rna']['top_conserved_genes']\n",
    "    conserved_df = add_log2fc(conserved_df)\n",
    "    conserved_df = conserved_df.sort_values(by='abs_rna_acc_log2fc_sum', ascending=True).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(conserved_df, num_conserved) # Get the location of the top num_conserved unique genes\n",
    "    conserved_df = conserved_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_conserved:\n",
    "        print(f'Warning: Only {num_unique} unique conserved genes found after filtering by promoter accessibility')\n",
    "\n",
    "    gene_list_dict = {'upregulated': upregulated_df, 'downregulated': downregulated_df, 'conserved': conserved_df}\n",
    "\n",
    "    return gene_list_dict\n",
    "    \n",
    "def add_log2fc(diff_df):\n",
    "    diff_df['accessibility_log2fc'] = np.log2(diff_df['experimental_max_accessibility'] / diff_df['control_max_accessibility'])\n",
    "    diff_df['rna_acc_log2fc_sum'] = diff_df['log2fc'] + diff_df['accessibility_log2fc']\n",
    "    diff_df['abs_rna_acc_log2fc_sum'] = diff_df['rna_acc_log2fc_sum'].abs()\n",
    "    return diff_df\n",
    "\n",
    "def get_promoter_accessibility(diff_df, config, lazy=True):\n",
    "    import zarr\n",
    "    # Load experimental and control zarr files\n",
    "    zarr_path_dict = {condition: [os.path.join(config['input_resource']['root'], config['input_resource']['atac'], f\"{rep}.zarr\") for rep in config['loci_source_config']['atac']['data_path'][condition].split(',')] for condition in ['experimental', 'control']}\n",
    "    if lazy:\n",
    "        zarr_dict = {condition: [zarr.open(zarr_pa, mode='r') for zarr_pa in zarr_path] for condition, zarr_path in zarr_path_dict.items()}\n",
    "    else:\n",
    "        print('Loading zarr files to memory')\n",
    "        zarr_dict = {condition: [load_zarr_to_memory(zarr_pa) for zarr_pa in zarr_path] for condition, zarr_path in zarr_path_dict.items()}\n",
    "\n",
    "    for zarr_condition, zarr_data in zarr_dict.items():\n",
    "        # Filter by promoter accessibility\n",
    "        max_accessibility_list = []\n",
    "        mean_accessibility_list = []\n",
    "        # Load promoter accessibility data by chromosome\n",
    "        print(f'Loading {zarr_condition} promoter accessibility data')\n",
    "        import tqdm\n",
    "        for row_idx, row in tqdm.tqdm(diff_df.iterrows(), total=len(diff_df)):\n",
    "            chrom = row['seqname']\n",
    "            start = row['tss_inference_start']\n",
    "            end = row['tss_inference_end']\n",
    "            max_accessibility_list.append(np.max(np.stack([zarr_da['chrs'][chrom][start:end] for zarr_da in zarr_data],axis=0)))\n",
    "            mean_accessibility_list.append(np.mean(np.stack([zarr_da['chrs'][chrom][start:end] for zarr_da in zarr_data],axis=0)))\n",
    "        diff_df[f'{zarr_condition}_max_accessibility'] = max_accessibility_list\n",
    "        diff_df[f'{zarr_condition}_mean_accessibility'] = mean_accessibility_list\n",
    "    return diff_df\n",
    "\n",
    "def load_zarr_to_memory(zarr_path):\n",
    "    import zarr\n",
    "    zarr_data = zarr.open(zarr_path, mode='r')\n",
    "    zarr_data_dict = {'chrs': {}}\n",
    "    print(f'Loading to memory: {zarr_path}')\n",
    "    for chrom in tqdm(zarr_data['chrs'].keys()):\n",
    "        zarr_data_dict['chrs'][chrom] = zarr_data['chrs'][chrom][:]\n",
    "    return zarr_data_dict\n",
    "\n",
    "def read_rna_data(rna_config):\n",
    "    diff_exp_df = pd.read_csv(rna_config['log2fc_csv'])\n",
    "    assert 'gene_name' in diff_exp_df.columns and 'experimental_exp' in diff_exp_df.columns and 'control_exp' in diff_exp_df.columns and 'log2fc' in diff_exp_df.columns\n",
    "    return diff_exp_df\n",
    "\n",
    "def add_loci(diff_exp_df, config):\n",
    "    # Add loci to the gene list\n",
    "    gff_path = os.path.join(config['input_resource']['root'], config['input_resource']['sequence'], config['preprocessing_data_config']['gff_file'])\n",
    "    diff_exp_df = get_gene_loci(diff_exp_df, gff_path)\n",
    "    tss_radius = config['loci_source_config']['rna']['tss_radius']\n",
    "    diff_exp_df = add_inference_loci(diff_exp_df, tss_radius, config)\n",
    "    return diff_exp_df\n",
    "\n",
    "def add_inference_loci(diff_exp_df, tss_radius, config):\n",
    "    diff_exp_df['tss_inference_start'] = diff_exp_df['tss'] - tss_radius\n",
    "    diff_exp_df['tss_inference_end'] = diff_exp_df['tss'] + tss_radius\n",
    "\n",
    "    # Clip to chromosome length\n",
    "    chr_sizes = load_chr_sizes(os.path.join(config['input_resource']['root'], config['input_resource']['sequence'], config['preprocessing_data_config']['chr_sizes_file']))\n",
    "\n",
    "    diff_exp_df['tss_inference_start'] = np.maximum(diff_exp_df['tss_inference_start'], 0)\n",
    "    diff_exp_df['tss_inference_end'] = np.minimum(diff_exp_df['tss_inference_end'], diff_exp_df['seqname'].map(chr_sizes))\n",
    "\n",
    "    return diff_exp_df\n",
    "\n",
    "def atac_split_filter_gene_type(diff_df, config):\n",
    "    # Add log2fc\n",
    "    diff_df['accessibility_log2fc'] = np.log2(diff_df['experimental_max_accessibility'] / diff_df['control_max_accessibility'])\n",
    "\n",
    "    # Filter by accessiblity\n",
    "    accessibility_threshold = config['loci_source_config']['atac']['active_locus_threshold']\n",
    "    cross_condition_max_accessibility = diff_df[['experimental_max_accessibility', 'control_max_accessibility']].max(axis=1)\n",
    "    select_idx = cross_condition_max_accessibility > accessibility_threshold\n",
    "    diff_df = diff_df[select_idx]\n",
    "\n",
    "    # Split by absolute log2fc\n",
    "    diff_df['abs_accessibility_log2fc'] = diff_df['accessibility_log2fc'].abs()\n",
    "    diff_df = diff_df.sort_values(by='abs_accessibility_log2fc', ascending=False)\n",
    "\n",
    "    # Get conserved genes\n",
    "    atac_config = config['loci_source_config']['atac']\n",
    "    num_conserved = atac_config['top_conserved_genes']\n",
    "    conserved_df = diff_df.copy().sort_values(by='abs_accessibility_log2fc', ascending=True).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(conserved_df, num_conserved) # Get the location of the top num_conserved unique genes\n",
    "    conserved_df = conserved_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_conserved:\n",
    "        print(f'Warning: Only {num_unique} unique conserved genes found after filtering by promoter accessibility')\n",
    "\n",
    "    # Get upregulated and downregulated genes\n",
    "    atac_threshold = atac_config['log2fc_threshold']\n",
    "    num_diff = atac_config['top_diff_genes']\n",
    "    upregulated_df = diff_df[diff_df['accessibility_log2fc'] > atac_threshold].copy().sort_values(by='accessibility_log2fc', ascending=False).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(upregulated_df, num_diff) # Get the location of the top num_diff unique genes\n",
    "    upregulated_df = upregulated_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_diff:\n",
    "        print(f'Warning: Only {num_unique} unique upregulated genes found after filtering by promoter accessibility')\n",
    "\n",
    "    downregulated_df = diff_df[diff_df['accessibility_log2fc'] < -atac_threshold].copy().sort_values(by='accessibility_log2fc', ascending=True).reset_index(drop=True)\n",
    "    filter_loc, num_unique = get_filter_location(downregulated_df, num_diff) # Get the location of the top num_diff unique genes\n",
    "    downregulated_df = downregulated_df.head(filter_loc).reset_index(drop=True)\n",
    "    if num_unique < num_diff:\n",
    "        print(f'Warning: Only {num_unique} unique downregulated genes found after filtering by promoter accessibility')\n",
    "\n",
    "    return {'upregulated': upregulated_df, 'downregulated': downregulated_df, 'conserved': conserved_df}\n",
    "\n",
    "def split_gene_type(diff_exp_df, rna_config):\n",
    "    # Split by absolute log2fc\n",
    "    diff_exp_df['abs_log2fc'] = diff_exp_df['log2fc'].abs()\n",
    "    diff_exp_df = diff_exp_df.sort_values(by='abs_log2fc', ascending=False)\n",
    "\n",
    "    # Get conserved genes\n",
    "    num_conserved = rna_config['top_conserved_genes'] * 4 # Leave room for atac-seq filtering\n",
    "    conserved_genes = diff_exp_df.sort_values(by='abs_log2fc', ascending=True).head(num_conserved).reset_index(drop=True)\n",
    "\n",
    "    # Get upregulated and downregulated genes\n",
    "    rna_threshold = rna_config['log2fc_threshold']\n",
    "    num_diff = rna_config['top_diff_genes'] * 2 # Leave room for atac-seq filtering\n",
    "    upregulated_genes = diff_exp_df[diff_exp_df['log2fc'] > rna_threshold].sort_values(by='log2fc', ascending=False).head(num_diff).reset_index(drop=True)\n",
    "    downregulated_genes = diff_exp_df[diff_exp_df['log2fc'] < -rna_threshold].sort_values(by='log2fc', ascending=True).head(num_diff).reset_index(drop=True)\n",
    "\n",
    "    return {'upregulated': upregulated_genes, 'downregulated': downregulated_genes, 'conserved': conserved_genes}\n",
    "\n",
    "def get_gene_loci(gene_df, gff_path):\n",
    "    # Find location of genes in the genome with a annotation file\n",
    "    print(f'Reading gff file from {gff_path}')\n",
    "    gff_df = read_genes(gff_path)\n",
    "    # Find the gene names in the annotation file\n",
    "    gene_names = gene_df['gene_name'].values\n",
    "    selected_gene_gff_df = []\n",
    "    print(f'Finding gene loci for {len(gene_names)} genes')\n",
    "    for gene_name in gene_names:\n",
    "        gene_gff_df = gff_df[gff_df['gene_name'] == gene_name]\n",
    "        selected_gene_gff_df.append(gene_gff_df)\n",
    "    selected_gene_gff_df = pd.concat(selected_gene_gff_df).reset_index(drop=True)\n",
    "    # Join the split_df with the gff_df\n",
    "    merged_df = gene_df.merge(selected_gene_gff_df, on='gene_name', how='left')\n",
    "\n",
    "    # Only keep gene on chr1-22 and X\n",
    "    merged_df = merged_df[merged_df['seqname'].isin([f'chr{i}' for i in range(1, 23)] + ['chrX'])]\n",
    "\n",
    "    # Remove genes with NA start or end and print warning\n",
    "    na_start = merged_df['start'].isna()\n",
    "    na_end = merged_df['end'].isna()\n",
    "    na_gene = merged_df[na_start | na_end]\n",
    "    if len(na_gene) > 0:\n",
    "        print(f'Warning: {len(na_gene)} genes with NA start or end')\n",
    "        print(na_gene['gene_name'].to_string(index=False))\n",
    "\n",
    "    merged_df = merged_df[~(na_start | na_end)]\n",
    "\n",
    "    # Add TSS column\n",
    "    merged_df['tss'] = np.where(merged_df['strand'] == '+', merged_df['start'], merged_df['end']).astype(int)\n",
    "\n",
    "    return merged_df\n",
    "\n",
    "def read_genes(gff_path):\n",
    "    names = ['seqname', 'source', 'feature', 'start', 'end', 'score', 'strand', 'frame', 'attribute']\n",
    "    genes = pd.read_csv(gff_path, sep='\\t', comment='#', names=names)\n",
    "    gene_names = []\n",
    "    transcript_types = []\n",
    "    transcript_names = []\n",
    "    for attribute in genes['attribute']:\n",
    "        try:\n",
    "            gene_field, gene_name = attribute.split(';')[5].split('=')\n",
    "            transcript_type_field, transcript_type = attribute.split(';')[6].split('=')\n",
    "            transcript_field, transcript_name = attribute.split(';')[7].split('=')\n",
    "            assert gene_field == 'gene_name'\n",
    "            assert transcript_type_field == 'transcript_type'\n",
    "            assert transcript_field == 'transcript_name'\n",
    "            gene_names.append(gene_name)\n",
    "            transcript_types.append(transcript_type)\n",
    "            transcript_names.append(transcript_name)\n",
    "        except Exception as e:\n",
    "            print(f'Error: {e}')\n",
    "            gene_names.append('')\n",
    "            transcript_names.append('')\n",
    "    genes['gene_name'] = gene_names\n",
    "    genes['transcript_name'] = transcript_names\n",
    "    genes['transcript_type'] = transcript_types\n",
    "    return genes\n",
    "\n",
    "def load_chr_sizes(chr_size_path):\n",
    "    chr_sizes = {}\n",
    "    with open(chr_size_path) as f:\n",
    "        for line in f:\n",
    "            chrom, size = line.strip().split()\n",
    "            chr_sizes[chrom] = int(size)\n",
    "    return chr_sizes\n",
    "\n",
    "def copy_merged_loci_to_inputs(config):\n",
    "    source_path = os.path.join(config[\"preprocessing_data_config\"][\"output\"][\"path\"], 'gene_list', 'merged_loci.bed')\n",
    "    dest_path = os.path.join(config[\"inference_config\"][\"input\"][\"root\"], config[\"inference_config\"][\"input\"][\"locus_list_path\"])\n",
    "    os.makedirs(os.path.dirname(dest_path), exist_ok=True)\n",
    "    import shutil\n",
    "    shutil.copy(source_path, dest_path)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "feae8583-5dfc-4275-acbd-3987215232ba",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "unexpected indent (1107164402.py, line 43)",
     "output_type": "error",
     "traceback": [
      "  \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[98]\u001b[39m\u001b[32m, line 43\u001b[39m\n\u001b[31m    \u001b[39m\u001b[31melif atac_seq_specific_config['mode'] == 'All_peaks':\u001b[39m\n    ^\n\u001b[31mIndentationError\u001b[39m\u001b[31m:\u001b[39m unexpected indent\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Load config\n",
    "with open('config.yaml', 'r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "\n",
    "# Get differential vs conserved gene lists\n",
    "# RNA-seq mode\n",
    "\n",
    "# ATAC-seq mode\n",
    "atac_seq_specific_config = config['loci_source_config']['atac_seq_specific_config']\n",
    "\n",
    "atac_config = config['loci_source_config']['atac']\n",
    "diff_acc_df = read_filter_gff_data(config)\n",
    "###read diff_acc_df\n",
    "gff_path = os.path.join(config['input_resource']['root'], config['input_resource']['sequence'], config['preprocessing_data_config']['gff_file'])\n",
    "print(f'Reading gff file from {gff_path}')\n",
    "gff_df = read_genes(gff_path)\n",
    "\n",
    "# Remove genes on chrX, chrY, and chrM\n",
    "gff_df = gff_df[~gff_df['seqname'].isin(['chrX', 'chrY', 'chrM'])]\n",
    "\n",
    "# Keep a few transcript_types\n",
    "transcript_types = ['protein_coding', 'lincRNA', 'miRNA', 'snRNA', 'snoRNA', 'rRNA']\n",
    "gff_df = gff_df[gff_df['transcript_type'].isin(transcript_types)]\n",
    "\n",
    "na_start = gff_df['start'].isna()\n",
    "na_end = gff_df['end'].isna()\n",
    "na_gene = gff_df[na_start | na_end]\n",
    "if len(na_gene) > 0:\n",
    "    print(f'Warning: {len(na_gene)} genes with NA start or end')\n",
    "    print(na_gene['gene_name'].to_string(index=False))\n",
    "\n",
    "gff_df = gff_df[~(na_start | na_end)]\n",
    "\n",
    "# Add TSS column\n",
    "gff_df['tss'] = np.where(gff_df['strand'] == '+', gff_df['start'], gff_df['end']).astype(int)\n",
    "\n",
    "tss_radius = config['loci_source_config']['atac_seq_specific_config']['promoter']['tss_radius']\n",
    "gff_df = add_inference_loci(gff_df, tss_radius, config)\n",
    "\n",
    "\n",
    "diff_acc_df = get_promoter_accessibility(diff_acc_df, config, lazy=False)\n",
    "gene_type_dict = atac_split_filter_gene_type(diff_acc_df, config)\n",
    "    elif atac_seq_specific_config['mode'] == 'All_peaks':\n",
    "        raise NotImplementedError('All_peaks mode not implemented')\n",
    "    else:\n",
    "        raise ValueError(f'Invalid mode: {config[\"loci_source_config\"][\"mode\"]}')\n",
    "else:\n",
    "    raise ValueError(f'Invalid mode: {config[\"loci_source_config\"][\"mode\"]}')\n",
    "\n",
    "# Save gene list and merged bed file for inference\n",
    "save_gene_list(gene_type_dict, config,config_name)\n",
    "\n",
    "# Copy merged_loci.bed to inputs folder\n",
    "copy_merged_loci_to_inputs(config,config_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (transformer)",
   "language": "python",
   "name": "transformer"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
