Mercurial > public > finance-parser
annotate process_document/app.py @ 5:2daf0dc08247
add get report endpoint
| author | Dennis C. M. <dennis@denniscm.com> |
|---|---|
| date | Mon, 05 Jun 2023 12:48:47 +0100 |
| parents | 9005b7590008 |
| children | d15ccf5f1373 |
| rev | line source |
|---|---|
| 2 | 1 import json |
| 2 import boto3 | |
| 3 from datetime import datetime | |
| 4 from collections import defaultdict | |
| 5 | |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
6 |
| 2 | 7 s3_client = boto3.client('s3') |
| 8 | |
| 9 | |
| 10 def lambda_handler(event, context): | |
| 4 | 11 event_msg = event['body']['message'] |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
12 |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
13 # Download file from s3 |
| 4 | 14 s3_client.download_file( |
| 15 event_msg['bucketName'], | |
| 16 event_msg['objectKey'], | |
| 17 '/tmp/document.json' | |
| 18 ) | |
| 2 | 19 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
20 with open('/tmp/document.json') as f: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
21 doc = json.load(f) |
| 2 | 22 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
23 # Analyze document |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
24 result = defaultdict(dict) |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
25 blocks = doc['Blocks'] |
| 2 | 26 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
27 # Get format |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
28 lines = filter_blocks(blocks, 'BlockType', 'LINE') |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
29 for line in lines: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
30 amount_format = get_format(line['Text']) |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
31 result['format'] = amount_format |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
32 if amount_format: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
33 break |
| 2 | 34 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
35 # Find dates value and position |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
36 data = defaultdict(dict) |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
37 cells = filter_blocks(blocks, 'BlockType', 'CELL') |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
38 for cell in cells: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
39 if not 'Relationships' in cell: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
40 continue |
| 2 | 41 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
42 child_ids = [r['Ids'] for r in cell['Relationships'] if r['Type'] == 'CHILD'][0] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
43 |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
44 # Get `Text` from `CELL` block |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
45 cell_text = '' |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
46 for index, child_id in enumerate(child_ids): |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
47 word_block = filter_blocks(blocks, 'Id', child_id)[0] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
48 cell_text += word_block['Text'] |
| 2 | 49 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
50 if index < len(child_ids) - 1: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
51 cell_text += '_' |
| 2 | 52 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
53 # Verify if `Text` could be a valid date |
| 4 | 54 date_string = is_date(clean_text(cell_text, 'date')) |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
55 if date_string: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
56 cell_text = date_string |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
57 result['dateRow'] = cell['RowIndex'] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
58 result['dateColumns'][cell['ColumnIndex']] = date_string |
| 2 | 59 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
60 cell_row_index = cell['RowIndex'] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
61 cell_column_index = cell['ColumnIndex'] |
| 4 | 62 data[cell_row_index][cell_column_index] = clean_text(cell_text) |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
63 |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
64 try: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
65 data[cell_row_index]['type'] = cell['EntityTypes'] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
66 except KeyError: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
67 pass |
| 2 | 68 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
69 # Delete unused row and columns |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
70 for row_index in list(data.keys()): |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
71 row = data[row_index] |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
72 for column_index in list(row.keys()): |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
73 if column_index not in result['dateColumns'] \ |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
74 and column_index != 1 and column_index != 'type': |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
75 del row[column_index] |
| 2 | 76 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
77 if len(row) > 1: |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
78 result['data'][row_index] = row |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
79 |
| 4 | 80 object_key = event_msg['objectKey'].replace('analyzed/', 'processed/') |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
81 data_string = json.dumps(result, indent=2, default=str) |
| 2 | 82 |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
83 s3_client.put_object( |
| 4 | 84 Bucket=event_msg['bucketName'], |
| 85 Key=object_key, | |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
86 Body=data_string |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
87 ) |
| 2 | 88 |
| 89 return { | |
| 90 "statusCode": 200, | |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
91 "body": { |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
92 "message": { |
| 4 | 93 "companyTicker": event_msg['companyTicker'], |
| 94 "docType": event_msg['docType'], | |
| 95 "fileId": event_msg['fileId'], | |
| 96 "fileName": event_msg['fileName'], | |
| 97 "objectKey": object_key, | |
| 98 "bucketName": event_msg['bucketName'] | |
|
3
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
99 } |
|
2e5f3664f3e4
documents analyzer almost finished
Dennis C. M. <dennis@denniscm.com>
parents:
2
diff
changeset
|
100 }, |
| 2 | 101 } |
| 102 | |
| 103 | |
| 104 def filter_blocks(blocks, block_key, block_value): | |
| 105 """ | |
| 106 Extract a block by key-value from array of blocks | |
| 107 """ | |
| 108 | |
| 109 return [block for block in blocks if block[block_key] == block_value] | |
| 110 | |
| 111 | |
| 112 def is_date(string_date): | |
| 113 """ | |
| 114 Verify if a string could be a date. | |
| 115 """ | |
| 116 | |
| 4 | 117 formats_allowed = ['%d-%m-%Y', '%d/%m/%Y', '%Y'] |
| 2 | 118 |
| 119 for format_allowed in formats_allowed: | |
| 120 try: | |
| 121 date = datetime.strptime(string_date, format_allowed) | |
| 122 | |
| 123 if date.year > datetime.now().year or date.year < 1900: | |
| 4 | 124 return # Date out of range date |
| 2 | 125 |
| 126 return date.strftime("%Y") | |
| 127 except ValueError: | |
| 4 | 128 continue |
| 2 | 129 |
| 130 return | |
| 131 | |
| 132 | |
| 133 def get_format(phrase): | |
| 134 """ | |
| 135 Given a phrase verify if it is specified the amount format | |
| 136 """ | |
| 137 | |
| 138 amount_formats = ['thousand', 'million', 'billion'] | |
| 139 | |
| 140 for amount_format in amount_formats: | |
| 141 plural_amount_format = f'{amount_format}s' | |
| 142 | |
| 143 if amount_format in phrase or plural_amount_format in phrase: | |
| 144 return amount_format | |
| 145 | |
| 146 | |
| 4 | 147 def clean_text(text, text_type='default'): |
| 2 | 148 """" |
| 149 Remove bad characters from word | |
| 150 """ | |
| 151 | |
| 4 | 152 special_chars = [ |
| 153 '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', | |
| 154 '-', '_', '+', '=', '[', ']', '{', '}', '\\', '|', | |
| 155 ';', ':', '"', '\'', '<', '>', '/', '?', '.', ',' | |
| 156 ] | |
| 157 | |
| 158 if text_type == 'date': | |
| 159 allowed_chars = ['_', '-', '/'] | |
| 2 | 160 |
| 4 | 161 # Sometimes date is '2020a' or 'b2020' because indexes |
| 162 if text[-1].isalpha(): | |
| 163 special_chars.append(text[-1]) | |
| 164 | |
| 165 if text[0].isalpha(): | |
| 166 special_chars.append(text[0]) | |
| 167 else: | |
| 168 allowed_chars = ['.', ',', '-', ' '] | |
| 169 | |
| 170 special_chars = [char for char in special_chars if char not in allowed_chars] | |
| 171 | |
| 172 for char in special_chars: | |
| 173 text = text.replace(char, '') | |
| 2 | 174 |
| 175 return text.lower() |