# pip install pdfplumber -q

import pandas as pd

import itertools

from pandas import DataFrame

import requests

import io

import pdfplumber

import sys

# sys.argv[0] is script name

infile = sys.argv[1]    # should be one column with a header

outfile = sys.argv[2]

startind = int(sys.argv[3])  # first index in Proceedings to process

endind = int(sys.argv[4])    # last index in Proceedings (if -1 go to end)

try:

    onlydo = sys.argv[5]    # 'Petitions', 'Responses', or 'Decisions'

except:                     # if only doing one kind

    onlydo = False

Proceedings = pd.read_csv(infile)

if endind == 0:

    Proceedings = Proceedings[Proceedings.columns[0]].to_list()[startind:]

else:

    Proceedings = Proceedings[Proceedings.columns[0]].to_list()[startind:endind+1]

def extractPDFText(pdfFile):

    # Find the first page with a numeric page number

    numpages = len(pdfFile.pages)

    if numpages < 11:

        start_page = 0

        end_page = min(10, numpages-1)

    elif numpages < 24:

        start_page = 0

        end_page = min(23, numpages-1)

    else:

        start_page = 0

        end_page = min(30, numpages-1)

    textr = ''

    for page in range(start_page,end_page+1):

        try:

            text = (pdfFile.pages[page]).extract_text()

            textr += text if text else ''

        except:

            pass

    return textr.replace('\n','')

Petitions = {}

Responses = {}

Decisions = {}

Petitions_json = {}

Responses_json = {}

Decisions_json = {}

for procnum in Proceedings:

    print('Processing', procnum)

    if (onlydo == 'Petitions') or (onlydo is False):

        rdoc = requests.get(f"https://developer.uspto.gov/ptab-api/documents?documentTypeName=Petition&proceedingNumber={procnum}")

        Petitions_json[procnum] = rdoc.json()['results']

        if rdoc.json()['recordTotalQuantity'] == 0:

            num_records = 0

            Petitions[procnum] = 'ERROR_NO_RECORDS_FOUND'

        else:

            doc_id = [result['documentIdentifier'] \

                      for result in Petitions_json[procnum] \

                      if result['documentTypeName']=='Petition']

            num_records = len(doc_id)

            if num_records == 0:

                doc_id = [result['documentIdentifier'] \

                          for result in Petitions_json[procnum] \

                          if ('Petition' in result['documentTitleText'] \

                              and not 'Rehear' in result['documentTitleText'])]

                num_records = len(doc_id)

            if num_records > 1:

                doc_id = [result['documentIdentifier']

                          for result in Petitions_json[procnum]

                          if result['documentTypeName']=='Petition' and

                          'Petition' in result['documentTitleText']]

                num_records = len(doc_id)

            if num_records == 1:

                doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                Petitions[procnum] = extractPDFText(pdfFile)

            elif num_records > 1:

                doc_id = [result['documentIdentifier']

                          for result in Petitions_json[procnum]

                          if (result['documentTypeName']=='Petition') \

                          and ('Petition' in result['documentTitleText']) \

                          and ('Corrected' in result['documentTitleText'])]

                num_records = len(doc_id)

                if num_records == 0:

                    doc_id = [result['documentIdentifier'] \

                              for result in Petitions_json[procnum] \

                              if ('Petition' in result['documentTitleText'] \

                                  and 'Corrected' in result['documentTitleText'])]

                num_records = len(doc_id)

                if num_records == 1:

                    doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                    pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                    Petitions[procnum] = extractPDFText(pdfFile)                

                else:

                    Petitions[procnum] = f'ERROR_TOO_MANY_RECORDS_RETURNED_{num_records}'

            elif num_records == 0:

                Petitions[procnum] = 'ERROR_NO_RECORD_RETURNED'

    if (onlydo == 'Responses') or (onlydo is False):

        rdoc1 = requests.get(f"https://developer.uspto.gov/ptab-api/documents?documentTypeName=Response&proceedingNumber={procnum}")

        rdoc2 = requests.get(f"https://developer.uspto.gov/ptab-api/documents?documentTypeName=Opposition&proceedingNumber={procnum}")

        Responses_json[procnum] = rdoc1.json()['results'] + rdoc2.json()['results']

        if (rdoc1.json()['recordTotalQuantity'] == 0) and (rdoc2.json()['recordTotalQuantity'] == 0):

            num_records = 0

            Responses[procnum] = 'ERROR_NO_RECORDS_FOUND'

        else:

            doc_id = [result['documentIdentifier']

                      for result in Responses_json[procnum] \

                      if 'Preliminary Response' in result['documentTypeName']]

            num_records = len(doc_id)

            if num_records == 0:

                doc_id = [result['documentIdentifier']

                          for result in Responses_json[procnum]

                          if ('Response' in result['documentTypeName'] or 'Opposition' in result['documentTypeName'] \

                          or result['documentTypeName']=='Notice') \

                          and ('Preliminary Response' in result['documentTitleText']) \

                          and ('Motion' not in result['documentTitleText'])]

                num_records = len(doc_id)

            if num_records == 1:

                doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                Responses[procnum] = extractPDFText(pdfFile)

            elif num_records > 1:

                doc_id = [result['documentIdentifier']

                          for result in Responses_json[procnum] \

                          if (result['documentTypeName']=='Response' or result['documentTypeName']=='Opposition') \

                          and ('Preliminary Response' in result['documentTitleText'] \

                               and 'Motion' not in result['documentTitleText'] \

                               and 'Exhibit' not in result['documentTitleText'])]

                num_records = len(doc_id)

                if num_records == 1:

                    doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                    pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                    Responses[procnum] = extractPDFText(pdfFile),

                elif num_records > 1:

                    doc_id = [result['documentIdentifier'] \

                              for result in Responses_json[procnum] \

                              if ('Preliminary Response' in result['documentTitleText']) \

                              and ('Corrected' in result['documentTitleText'])]

                    num_records = len(doc_id)

                    if num_records > 1:

                        Responses[procnum] = f'WARNING_TOO_MANY_RECORDS_RETURNED_{num_records}_'

                        doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                        pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                        Responses[procnum] += extractPDFText(pdfFile)

                elif num_records == 0:

                    doc_id = [result['documentIdentifier'] \

                              for result in Responses_json[procnum] \

                              if 'Preliminary Response' in result['documentTitleText']]

                    num_records = len(doc_id)

                    if num_records == 0:

                        doc_id = [result['documentIdentifier'] \

                                  for result in Responses_json[procnum] \

                                  if ('Response' in result['documentTypeName']) \

                                  and ('Motion' not in result['documentTitleText']) \

                                  and ('Exhibit' not in result['documentTitleText'])]

                        num_records = len(doc_id)

                        if num_records > 1:

                            doc_id = [result['documentIdentifier'] \

                                      for result in Responses_json[procnum] \

                                      if ('Response' in result['documentTypeName']) \

                                      and ('Response' in result['documentTitleText']) \

                                      and ('Exhibit' not in result['documentTitleText'])]

                            num_records = len(doc_id)

                    if num_records == 1:

                        doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                        pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                        Responses[procnum] = extractPDFText(pdfFile)

                    elif num_records == 0:

                        Responses[procnum] = 'ERROR_NO_RECORD_RETURNED'

                    elif num_records > 1:

                        doc_id = [result['documentIdentifier'] \

                                  for result in Responses_json[procnum] \

                                  if ('Preliminary Response' in result['documentTitleText']) \

                                  and ('Corrected' in result['documentTitleText'])]

                        num_records = len(doc_id)

                        if num_records == 0:

                            doc_id = [result['documentIdentifier'] \

                                      for result in Responses_json[procnum] \

                                      if ('Response' in result['documentTypeName']) \

                                      and ('Response' in result['documentTitleText']) \

                                      and ('Exhibit' not in result['documentTitleText'])]

                            num_records = len(doc_id)

                        if num_records > 1:

                            Responses[procnum] = f'WARNING_TOO_MANY_RECORDS_RETURNED_{num_records}_'

                            doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                            pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                            Responses[procnum] += extractPDFText(pdfFile)

                        if num_records == 1:

                            doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                            pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                            Responses[procnum] = extractPDFText(pdfFile)

            elif num_records == 0:

                Responses[procnum] = 'ERROR_NO_RECORD_RETURNED'

    if (onlydo == 'Decisions') or (onlydo is False):

        rdoc = requests.get(f"https://developer.uspto.gov/ptab-api/documents?documentTypeName=Decision&proceedingNumber={procnum}")

        Decisions_json[procnum] = rdoc.json()['results']

        if rdoc.json()['recordTotalQuantity'] == 0:

            num_records = 0

            rdoc = requests.get(f"https://developer.uspto.gov/ptab-api/documents?documentTypeName=Notice&proceedingNumber={procnum}")

            doc_id = [result['documentIndentifier'] for result in rdoc \

                      if ('Institution' in result['documentTitleText'] \

                      and 'Decision' in result['documentTitleText']) \

                      and ('Denying Rehearing' not in result['documentTitleText'] \

                           or 'Seal' not in result['documentTitleText'] \

                           or 'Motion' not in result['documentTitleText'])]

            num_records = len(doc_id)

            if (num_records == 0) or (num_records > 1):

                Decisions[procnum] = 'ERROR_NO_RECORDS_FOUND'

            else:

                doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                Decisions[procnum] = extractPDFText(pdfFile)

        else:

            doc_id = [result['documentIdentifier']

                      for result in Decisions_json[procnum]

                      if result['documentTypeName']=='Institution Decision']

            num_records = len(doc_id)

            if num_records > 1:

                doc_id = [result['documentIdentifier']

                          for result in Decisions_json[procnum]

                          if (result['documentTypeName']=='Institution Decision' \

                              or result['documentTypeName']=='Decision Granting Institution' \

                              or result['documentTypeName']=='Decision Denying Institution') \

                          and ('Decision' in result['documentTitleText'])]

                num_records = len(doc_id)

                if num_records > 1:

                    doc_id = [result['documentIdentifier']

                              for result in Decisions_json[procnum]

                              if (result['documentTypeName']=='Institution Decision' \

                                  or result['documentTypeName']=='Decision Granting Institution' \

                                  or result['documentTypeName']=='Decision Denying Institution') \

                              and ('Decision' in result['documentTitleText'] and not 'Rehearing' in result['documentTitleText'])]

                    num_records = len(doc_id)

            if num_records == 1:

                doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                Decisions[procnum] = extractPDFText(pdfFile)

            elif num_records > 1:

                Decisions[procnum] = f'ERROR_TOO_MANY_RECORDS_RETURNED_{num_records}'

            elif num_records == 0:

                doc_id = [result['documentIdentifier'] \

                          for result in Decisions_json[procnum] \

                          if ('Decision' in result['documentTitleText']) \

                          and ('Institution' in result['documentTitleText'])]

                num_records = len(doc_id)

                if num_records == 0:

                    Decisions[procnum] = 'ERROR_NO_RECORD_RETURNED'

                elif num_records == 1:

                    doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                    pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                    Decisions[procnum] = extractPDFText(pdfFile)                

                elif num_records > 1:

                    doc_id = [result['documentIdentifier'] \

                              for result in Decisions_json[procnum] \

                              if ('Decision' in result['documentTitleText']) \

                              and ('Institution' in result['documentTitleText']) \

                              and ('Rehearing' not in result['documentTitleText'])]

                    num_records = len(doc_id)

                    if num_records == 1:

                        doc = requests.get(f"https://developer.uspto.gov/ptab-api/documents/{doc_id[0]}/download",stream=True)

                        pdfFile = pdfplumber.open(io.BytesIO(doc.content))

                        Decisions[procnum] = extractPDFText(pdfFile)

                    else:

                        Decisions[procnum] = f'ERROR_TOO_MANY_RECORDS_RETURNED_{num_records}'

if onlydo == 'Petitions':

    ResultsDF = DataFrame.from_dict({key:[Petitions[key]]

                                     for key in Proceedings},

                                    orient='index',

                                    columns=['Petitions'])

elif onlydo == 'Responses':

    ResultsDF = DataFrame.from_dict({key:[Responses[key]]

                                     for key in Responses},

                                    orient='index',

                                    columns=['Responses'])

elif onlydo == 'Decisions':

    ResultsDF = DataFrame.from_dict({key:[Decisions[key]]

                                     for key in Proceedings},

                                    orient='index',

                                    columns=['Decisions'])

else:

    ResultsDF = DataFrame.from_dict({key:[Petitions[key], Responses[key], Decisions[key]]

                                     for key in Proceedings},

                                    orient='index',

                                    columns=['Petitions','Responses','Decisions'])

ResultsDF = ResultsDF.reset_index().rename(columns={'index':'Proceeding'})

ResultsDF.to_csv(outfile, sep='\t', index=False)