second pass

res2.txt

@@ -0,0 +1,340 @@
+Output exceeds the size limit. Open the full output data in a text editor
+
+table: 2-1137692-1
+columns: Entrant,Constructor,Chassis,Engine †,Tyre,Driver,Rounds
+Q: What were the rounds on the Engine † of the Ferrari 048?
+A: SELECT  Rounds FROM 2-1137692-1 WHERE Engine † = 'ferrari 048'
+END
+
+
+table: 1-21530474-1
+columns: Chassis code,Model no.,Production years,Drivetrain,Transmission,Engine type,Engine code,Region(s)
+Q: Name the drivetrain for 1ur-fse for usf41
+A: SELECT  Drivetrain FROM 1-21530474-1 WHERE Engine code = '1UR-FSE' AND Chassis code = 'USF41'
+END
+
+
+table: 2-14155087-1
+columns: Callsign,Area served,Frequency,Band,On-air ID,Purpose
+Q: What is the Callsign with an Area of tamworth and frequency of 0 88.9?
+A: SELECT  Callsign FROM 2-14155087-1 WHERE Area served = 'tamworth' AND Frequency = '0 88.9'
+END
+
+
+table: 2-17580726-2
+columns: Date,Opponent,Venue,Score,Attendance,Scorers
+Q: What is the number of people in attendance when Tonbridge Angels is the opponent?
+...
+Q: What were the match points when Bordeaux-Bègles was eliminated from competition? 
+A: SELECT  Match points FROM 1-27986200-3 WHERE Eliminated from competition = 'Bordeaux-Bègles'
+END
+
+/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda did not contain libcudart.so as expected! Searching further paths...
+  warn(msg)
+The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. 
+The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. 
+The class this function is called from is 'LlamaTokenizer'.
+
+===================================BUG REPORT===================================
+Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
+================================================================================
+CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so
+CUDA SETUP: Highest compute capability among GPUs detected: 7.5
+CUDA SETUP: Detected CUDA version 113
+CUDA SETUP: Loading binary /home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/bitsandbytes/libbitsandbytes_cuda113.so...
+True
+92
+                  0
+count  56355.000000
+mean     101.219519
+std       21.740325
+min       63.000000
+25%       87.500000
+50%       97.000000
+75%      109.000000
+max      461.000000
+32084
+ [250/250 3:49:26, Epoch 0/1]
+Step	Training Loss
+1	2.748800
+2	2.699100
+3	2.670200
+4	2.600500
+5	2.560100
+6	2.556800
+7	2.498100
+8	2.515400
+9	2.436100
+10	2.411700
+11	2.346400
+12	2.276300
+13	2.238000
+14	2.189100
+15	2.109200
+16	2.058000
+17	1.983900
+18	1.928600
+19	1.824100
+20	1.794700
+21	1.681200
+22	1.598900
+23	1.562000
+24	1.527200
+25	1.518700
+26	1.493100
+27	1.500500
+28	1.464000
+29	1.386900
+30	1.373400
+31	1.362200
+32	1.360800
+33	1.321000
+34	1.310500
+35	1.302600
+36	1.256100
+37	1.252500
+38	1.202300
+39	1.249100
+40	1.188600
+41	1.203200
+42	1.150000
+43	1.182000
+44	1.192300
+45	1.133100
+46	1.119600
+47	1.097000
+48	1.142100
+49	1.117200
+50	1.129200
+51	1.087300
+52	1.098700
+53	1.135400
+54	1.071700
+55	1.087300
+56	1.051400
+57	1.068300
+58	1.092500
+59	1.068600
+60	1.072800
+61	1.074000
+62	1.060400
+63	1.065800
+64	1.075900
+65	1.059500
+66	1.039600
+67	1.051400
+68	1.049500
+69	1.023800
+70	1.071900
+71	1.051000
+72	1.034700
+73	1.041600
+74	1.030900
+75	1.010800
+76	1.019800
+77	1.005000
+78	1.043800
+79	1.009200
+80	1.017100
+81	1.044600
+82	1.022600
+83	1.011400
+84	0.996600
+85	1.029900
+86	0.988200
+87	1.005600
+88	0.986600
+89	1.025300
+90	1.012500
+91	0.988100
+92	1.001800
+93	0.987100
+94	1.017600
+95	0.998500
+96	0.966600
+97	0.983700
+98	0.961800
+99	0.969000
+100	0.989200
+101	0.956400
+102	0.976000
+103	1.000100
+104	1.001500
+105	0.995900
+106	0.989700
+107	0.965700
+108	0.968400
+109	1.019600
+110	1.000100
+111	0.978500
+112	0.978900
+113	0.952600
+114	0.975400
+115	0.989400
+116	0.968500
+117	0.960100
+118	0.979100
+119	0.955100
+120	0.934800
+121	0.943600
+122	0.976700
+123	0.998700
+124	0.930500
+125	0.953500
+126	0.978000
+127	0.967300
+128	0.929400
+129	0.963100
+130	0.961500
+131	0.978500
+132	0.937200
+133	0.953400
+134	0.962000
+135	0.950700
+136	0.925100
+137	0.958800
+138	0.926200
+139	0.930600
+140	0.968900
+141	0.970400
+142	0.927100
+143	0.911800
+144	0.953200
+145	0.907100
+146	0.935900
+147	0.970600
+148	0.920400
+149	0.930200
+150	0.926700
+151	0.913400
+152	0.926800
+153	0.967200
+154	0.939500
+155	0.910600
+156	0.926400
+157	0.935400
+158	0.967700
+159	0.899000
+160	0.916600
+161	0.961600
+162	0.898200
+163	0.944600
+164	0.935700
+165	0.922500
+166	0.897600
+167	0.968600
+168	0.927400
+169	0.910900
+170	0.904700
+171	0.899800
+172	0.896400
+173	0.862100
+174	0.909100
+175	0.903200
+176	0.958600
+177	0.902500
+178	0.894900
+179	0.937900
+180	0.900700
+181	0.922300
+182	0.939300
+183	0.932600
+184	0.913300
+185	0.941700
+186	0.886300
+187	0.918000
+188	0.884000
+189	0.947400
+190	0.894500
+191	0.929300
+192	0.877300
+193	0.894300
+194	0.867800
+195	0.913500
+196	0.908100
+197	0.931200
+198	0.911000
+199	0.941800
+200	0.913000
+201	0.921800
+202	0.921700
+203	0.914500
+204	0.910500
+205	0.906600
+206	0.915100
+207	0.881600
+208	0.884700
+209	0.902900
+210	0.882600
+211	0.891000
+212	0.914400
+213	0.930400
+214	0.891100
+215	0.859300
+216	0.891800
+217	0.873000
+218	0.925900
+219	0.905700
+220	0.921200
+221	0.890200
+222	0.915800
+223	0.887300
+224	0.898300
+225	0.865600
+226	0.873900
+227	0.904800
+228	0.917900
+229	0.923400
+230	0.939700
+231	0.913400
+232	0.873100
+233	0.896700
+234	0.892100
+235	0.902100
+236	0.927200
+237	0.912900
+238	0.872900
+239	0.904700
+240	0.879600
+241	0.879800
+242	0.908800
+243	0.909800
+244	0.838400
+245	0.889200
+246	0.912900
+247	0.879700
+248	0.910700
+249	0.845400
+250	0.882200
+/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+Output exceeds the size limit. Open the full output data in a text editor
+from model
+<unk>table: 1-12028543-3
+columns: Season,Cup FinalDate,WinningTeam,Score,LosingTeam,Location,Cup Final Attendance
+Q: Who was the winning team in the 1989 season?
+A:  SELECT  WinningTeam FROM 1-12028543-3 WHERE Season = '1989'
+END
+END
+END
+END
+
+expected answer
+SELECT  WinningTeam FROM 1-12028543-3 WHERE Season = '1989'
+END
+
+from model
+<unk>table: 2-18096431-5
+columns: Place,Player,Country,Score,To par
+Q: What is To par, when Country is "United States", and when Player is "Mark Brooks"?
+A: 18-1
+END
+
+
+expected answer
+SELECT  To par FROM 2-18096431-5 WHERE Country = 'united states' AND Player = 'mark brooks'
+END
+...
+expected answer
+SELECT  Score FROM 2-17978030-6 WHERE Set 3 = '26–28'
+END

sqllama-out2/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ee15525f45ab11e3e7ba334c0639b7263ea25ae0d42aa22f801022020ffc493
+oid sha256:423e1095c473c661c7ce62b23e3c06bb80780c572177c2ab77ef1451117cf83d
 size 8434381

wikisql.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -11,7 +11,7 @@
        "True"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 1,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -55,7 +55,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a9428ee09f334655b6b261d478cbd3d0",
+       "model_id": "0b970b9989854c33aa4d19fa9457d7a2",
        "version_major": 2,
        "version_minor": 0
       },
@@ -96,30 +96,40 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "table: 2-13081928-2\n",
-      "columns: Country,Chart,Period,Peak position,Sales\n",
-      "Q: Name the period for Chart of g-music j-pop/k-pop chart\n",
-      "A: SELECT  Period FROM 2-13081928-2 WHERE Chart = 'g-music j-pop/k-pop chart'\n",
-      "\n",
-      "table: 2-13612447-1\n",
-      "columns: Fraction,Ellipsis,Vinculum,Dots,Parentheses\n",
-      "Q: What is the dot value when the ellipsis is 0.012345679…?\n",
-      "A: SELECT  Dots FROM 2-13612447-1 WHERE Ellipsis = '0.012345679…'\n",
-      "\n",
-      "table: 1-168274-1\n",
-      "columns: Company,ICB Sector,Ticker symbol,Index weighting (%) at 17 January 2013,Market cap. at April 2013 (€)\n",
-      "Q: Name the total number of index weighting % at 17 january 2013 for bouygues\n",
-      "A: SELECT COUNT Index weighting (%) at 17 January 2013 FROM 1-168274-1 WHERE Company = 'Bouygues'\n",
-      "\n",
-      "table: 2-15826191-2\n",
-      "columns: Rank,Nation,Gold,Silver,Bronze,Total\n",
-      "Q: What is the lowest gold when there are 0 bronze and the total is less than 2, and silver is less than 0?\n",
-      "A: SELECT MIN Gold FROM 2-15826191-2 WHERE Bronze = 0 AND Total < 2 AND Silver < 0\n",
-      "\n",
-      "table: 2-16387912-1\n",
-      "columns: Home team,Home team score,Away team,Away team score,Ground,Date,Time\n",
-      "Q: What is Ground, when Away Team is Sydney?\n",
-      "A: SELECT  Ground FROM 2-16387912-1 WHERE Away team = 'sydney'\n"
+      "table: 2-1137692-1\n",
+      "columns: Entrant,Constructor,Chassis,Engine †,Tyre,Driver,Rounds\n",
+      "Q: What were the rounds on the Engine † of the Ferrari 048?\n",
+      "A: SELECT  Rounds FROM 2-1137692-1 WHERE Engine † = 'ferrari 048'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 1-21530474-1\n",
+      "columns: Chassis code,Model no.,Production years,Drivetrain,Transmission,Engine type,Engine code,Region(s)\n",
+      "Q: Name the drivetrain for 1ur-fse for usf41\n",
+      "A: SELECT  Drivetrain FROM 1-21530474-1 WHERE Engine code = '1UR-FSE' AND Chassis code = 'USF41'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 2-14155087-1\n",
+      "columns: Callsign,Area served,Frequency,Band,On-air ID,Purpose\n",
+      "Q: What is the Callsign with an Area of tamworth and frequency of 0 88.9?\n",
+      "A: SELECT  Callsign FROM 2-14155087-1 WHERE Area served = 'tamworth' AND Frequency = '0 88.9'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 2-17580726-2\n",
+      "columns: Date,Opponent,Venue,Score,Attendance,Scorers\n",
+      "Q: What is the number of people in attendance when Tonbridge Angels is the opponent?\n",
+      "A: SELECT  Attendance FROM 2-17580726-2 WHERE Opponent = 'tonbridge angels'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 1-27986200-3\n",
+      "columns: Proceed to Quarter-final,Match points,Aggregate score,Points margin,Eliminated from competition\n",
+      "Q: What were the match points when Bordeaux-Bègles was eliminated from competition? \n",
+      "A: SELECT  Match points FROM 1-27986200-3 WHERE Eliminated from competition = 'Bordeaux-Bègles'\n",
+      "END\n",
+      "\n"
      ]
     }
    ],
@@ -228,17 +238,17 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "89\n",
+      "92\n",
       "                  0\n",
       "count  56355.000000\n",
-      "mean      98.219519\n",
+      "mean     101.219519\n",
       "std       21.740325\n",
-      "min       60.000000\n",
-      "25%       84.500000\n",
-      "50%       94.000000\n",
-      "75%      106.000000\n",
-      "max      458.000000\n",
-      "35608\n"
+      "min       63.000000\n",
+      "25%       87.500000\n",
+      "50%       97.000000\n",
+      "75%      109.000000\n",
+      "max      461.000000\n",
+      "32084\n"
      ]
     }
    ],
@@ -262,18 +272,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d548eb2af20f435fa1af81e9045a2d0e",
+       "model_id": "01debeeb68bb40a7b83031e88c5ace1e",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "Map:   0%|          | 0/1000 [00:00<?, ? examples/s]"
+       "Map:   0%|          | 0/32084 [00:00<?, ? examples/s]"
       ]
      },
      "metadata": {},
@@ -282,9 +292,8 @@
    ],
    "source": [
     "import random, datasets\n",
-    "d = {'prompt': random.sample(data_red, 1000)}\n",
-    "\n",
-    "tokenizer.pad_token_id = tokenizer.eos_token\n",
+    "#d = {'prompt': random.sample(data_red, 1000)}\n",
+    "d = {'prompt': data_red}\n",
     "\n",
     "data = datasets.Dataset.from_dict(d)\n",
     "data = data.map(lambda x:\n",
@@ -300,7 +309,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -344,7 +353,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -353,8 +362,8 @@
        "\n",
        "    <div>\n",
        "      \n",
-       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [7/7 05:33, Epoch 0/1]\n",
+       "      <progress value='250' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [250/250 3:49:26, Epoch 0/1]\n",
        "    </div>\n",
        "    <table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
@@ -366,104 +375,1224 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <td>1</td>\n",
-       "      <td>2.710700</td>\n",
+       "      <td>2.748800</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>2</td>\n",
-       "      <td>2.680400</td>\n",
+       "      <td>2.699100</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>3</td>\n",
-       "      <td>2.684500</td>\n",
+       "      <td>2.670200</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>4</td>\n",
-       "      <td>2.625600</td>\n",
+       "      <td>2.600500</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>5</td>\n",
-       "      <td>2.609600</td>\n",
+       "      <td>2.560100</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>6</td>\n",
-       "      <td>2.619100</td>\n",
+       "      <td>2.556800</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>7</td>\n",
-       "      <td>2.603800</td>\n",
+       "      <td>2.498100</td>\n",
        "    </tr>\n",
-       "  </tbody>\n",
-       "</table><p>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "trainer = transformers.Trainer(\n",
-    "    model = model,\n",
-    "    train_dataset = data,\n",
-    "    args = targs,\n",
-    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
-    ")\n",
-    "trainer.train(resume_from_checkpoint=False)\n",
-    "model.save_pretrained('sqllama-out2')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/transformers/generation/utils.py:1220: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
-      "  \"You have modified the pretrained model configuration to control generation. This is a\"\n",
-      "/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
-      "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "from model\n",
-      " ⁇  table: 1-25800134-1\n",
-      "columns: Series #,Season #,Title,Director,Writer(s),Airdate\n",
-      "Q: Who wrote the episode with series number 56?\n",
-      "A: 56-101, \"The Cage\", Gene Roddenberry\n",
-      "Q: Who wrote the episode with series number 56? (2)\n",
-      "A: 56-101,\n",
-      "expected answer SELECT  Writer(s) FROM 1-25800134-1 WHERE Series # = 56\n"
-     ]
-    }
-   ],
-   "source": [
-    "def get_query(q):\n",
-    "    \n",
-    "    toks = tokenizer(q , return_tensors='pt')\n",
-    "    ctoks = toks.input_ids.to('cuda')\n",
-    "    gen = model.generate(ctoks, max_length=100)\n",
-    "    return tokenizer.decode(gen[0])\n",
-    "\n",
-    "M = len(q_red)\n",
-    "j = random.randint(0,M-1)\n",
-    "qs = q_red[j]\n",
-    "a = a_red[j]\n",
-    "\n",
-    "ma = get_query(qs)\n",
-    "\n",
-    "#print(qs)\n",
-    "print('from model')\n",
-    "print(ma)\n",
-    "print\n",
-    "print('expected answer',a)\n"
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>2.515400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>2.436100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>2.411700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>2.346400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>2.276300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>2.238000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>2.189100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15</td>\n",
+       "      <td>2.109200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16</td>\n",
+       "      <td>2.058000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17</td>\n",
+       "      <td>1.983900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18</td>\n",
+       "      <td>1.928600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19</td>\n",
+       "      <td>1.824100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>1.794700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>21</td>\n",
+       "      <td>1.681200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>22</td>\n",
+       "      <td>1.598900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>23</td>\n",
+       "      <td>1.562000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>24</td>\n",
+       "      <td>1.527200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>1.518700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>26</td>\n",
+       "      <td>1.493100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>27</td>\n",
+       "      <td>1.500500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>28</td>\n",
+       "      <td>1.464000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>29</td>\n",
+       "      <td>1.386900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>1.373400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>31</td>\n",
+       "      <td>1.362200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>32</td>\n",
+       "      <td>1.360800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>33</td>\n",
+       "      <td>1.321000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>34</td>\n",
+       "      <td>1.310500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>35</td>\n",
+       "      <td>1.302600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>36</td>\n",
+       "      <td>1.256100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>37</td>\n",
+       "      <td>1.252500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>38</td>\n",
+       "      <td>1.202300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>39</td>\n",
+       "      <td>1.249100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>1.188600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>41</td>\n",
+       "      <td>1.203200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>42</td>\n",
+       "      <td>1.150000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>43</td>\n",
+       "      <td>1.182000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>44</td>\n",
+       "      <td>1.192300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>45</td>\n",
+       "      <td>1.133100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>46</td>\n",
+       "      <td>1.119600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>47</td>\n",
+       "      <td>1.097000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>48</td>\n",
+       "      <td>1.142100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>49</td>\n",
+       "      <td>1.117200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>1.129200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>51</td>\n",
+       "      <td>1.087300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>52</td>\n",
+       "      <td>1.098700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>53</td>\n",
+       "      <td>1.135400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>54</td>\n",
+       "      <td>1.071700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>55</td>\n",
+       "      <td>1.087300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>56</td>\n",
+       "      <td>1.051400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>57</td>\n",
+       "      <td>1.068300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>58</td>\n",
+       "      <td>1.092500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>59</td>\n",
+       "      <td>1.068600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>1.072800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>61</td>\n",
+       "      <td>1.074000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>62</td>\n",
+       "      <td>1.060400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>63</td>\n",
+       "      <td>1.065800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>64</td>\n",
+       "      <td>1.075900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>65</td>\n",
+       "      <td>1.059500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>66</td>\n",
+       "      <td>1.039600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>67</td>\n",
+       "      <td>1.051400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>68</td>\n",
+       "      <td>1.049500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>69</td>\n",
+       "      <td>1.023800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>1.071900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>71</td>\n",
+       "      <td>1.051000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>72</td>\n",
+       "      <td>1.034700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>73</td>\n",
+       "      <td>1.041600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>74</td>\n",
+       "      <td>1.030900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>75</td>\n",
+       "      <td>1.010800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>76</td>\n",
+       "      <td>1.019800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>77</td>\n",
+       "      <td>1.005000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>78</td>\n",
+       "      <td>1.043800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>79</td>\n",
+       "      <td>1.009200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>1.017100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>81</td>\n",
+       "      <td>1.044600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>82</td>\n",
+       "      <td>1.022600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>83</td>\n",
+       "      <td>1.011400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>84</td>\n",
+       "      <td>0.996600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>85</td>\n",
+       "      <td>1.029900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>86</td>\n",
+       "      <td>0.988200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>87</td>\n",
+       "      <td>1.005600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>88</td>\n",
+       "      <td>0.986600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>89</td>\n",
+       "      <td>1.025300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>1.012500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>91</td>\n",
+       "      <td>0.988100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>92</td>\n",
+       "      <td>1.001800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>93</td>\n",
+       "      <td>0.987100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>94</td>\n",
+       "      <td>1.017600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>95</td>\n",
+       "      <td>0.998500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>96</td>\n",
+       "      <td>0.966600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>97</td>\n",
+       "      <td>0.983700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>98</td>\n",
+       "      <td>0.961800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>99</td>\n",
+       "      <td>0.969000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>0.989200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>101</td>\n",
+       "      <td>0.956400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>102</td>\n",
+       "      <td>0.976000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>103</td>\n",
+       "      <td>1.000100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>104</td>\n",
+       "      <td>1.001500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>105</td>\n",
+       "      <td>0.995900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>106</td>\n",
+       "      <td>0.989700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>107</td>\n",
+       "      <td>0.965700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>108</td>\n",
+       "      <td>0.968400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>109</td>\n",
+       "      <td>1.019600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>1.000100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>111</td>\n",
+       "      <td>0.978500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>112</td>\n",
+       "      <td>0.978900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>113</td>\n",
+       "      <td>0.952600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>114</td>\n",
+       "      <td>0.975400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>115</td>\n",
+       "      <td>0.989400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>116</td>\n",
+       "      <td>0.968500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>117</td>\n",
+       "      <td>0.960100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>118</td>\n",
+       "      <td>0.979100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>119</td>\n",
+       "      <td>0.955100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>0.934800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>121</td>\n",
+       "      <td>0.943600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>122</td>\n",
+       "      <td>0.976700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>123</td>\n",
+       "      <td>0.998700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>124</td>\n",
+       "      <td>0.930500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>125</td>\n",
+       "      <td>0.953500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>126</td>\n",
+       "      <td>0.978000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>127</td>\n",
+       "      <td>0.967300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>128</td>\n",
+       "      <td>0.929400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>129</td>\n",
+       "      <td>0.963100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>0.961500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>131</td>\n",
+       "      <td>0.978500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>132</td>\n",
+       "      <td>0.937200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>133</td>\n",
+       "      <td>0.953400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>134</td>\n",
+       "      <td>0.962000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>135</td>\n",
+       "      <td>0.950700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>136</td>\n",
+       "      <td>0.925100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>137</td>\n",
+       "      <td>0.958800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>138</td>\n",
+       "      <td>0.926200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>139</td>\n",
+       "      <td>0.930600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>0.968900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>141</td>\n",
+       "      <td>0.970400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>142</td>\n",
+       "      <td>0.927100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>143</td>\n",
+       "      <td>0.911800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>144</td>\n",
+       "      <td>0.953200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>145</td>\n",
+       "      <td>0.907100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>146</td>\n",
+       "      <td>0.935900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>147</td>\n",
+       "      <td>0.970600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>148</td>\n",
+       "      <td>0.920400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>149</td>\n",
+       "      <td>0.930200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>0.926700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>151</td>\n",
+       "      <td>0.913400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>152</td>\n",
+       "      <td>0.926800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>153</td>\n",
+       "      <td>0.967200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>154</td>\n",
+       "      <td>0.939500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>155</td>\n",
+       "      <td>0.910600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>156</td>\n",
+       "      <td>0.926400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>157</td>\n",
+       "      <td>0.935400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>158</td>\n",
+       "      <td>0.967700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>159</td>\n",
+       "      <td>0.899000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>0.916600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>161</td>\n",
+       "      <td>0.961600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>162</td>\n",
+       "      <td>0.898200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>163</td>\n",
+       "      <td>0.944600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>164</td>\n",
+       "      <td>0.935700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>165</td>\n",
+       "      <td>0.922500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>166</td>\n",
+       "      <td>0.897600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>167</td>\n",
+       "      <td>0.968600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>168</td>\n",
+       "      <td>0.927400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>169</td>\n",
+       "      <td>0.910900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>0.904700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>171</td>\n",
+       "      <td>0.899800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>172</td>\n",
+       "      <td>0.896400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>173</td>\n",
+       "      <td>0.862100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>174</td>\n",
+       "      <td>0.909100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>175</td>\n",
+       "      <td>0.903200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>176</td>\n",
+       "      <td>0.958600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>177</td>\n",
+       "      <td>0.902500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>178</td>\n",
+       "      <td>0.894900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>179</td>\n",
+       "      <td>0.937900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>0.900700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>181</td>\n",
+       "      <td>0.922300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>182</td>\n",
+       "      <td>0.939300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>183</td>\n",
+       "      <td>0.932600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>184</td>\n",
+       "      <td>0.913300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>185</td>\n",
+       "      <td>0.941700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>186</td>\n",
+       "      <td>0.886300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>187</td>\n",
+       "      <td>0.918000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>188</td>\n",
+       "      <td>0.884000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>189</td>\n",
+       "      <td>0.947400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>0.894500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>191</td>\n",
+       "      <td>0.929300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>192</td>\n",
+       "      <td>0.877300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>193</td>\n",
+       "      <td>0.894300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>194</td>\n",
+       "      <td>0.867800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>195</td>\n",
+       "      <td>0.913500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>196</td>\n",
+       "      <td>0.908100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>197</td>\n",
+       "      <td>0.931200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>198</td>\n",
+       "      <td>0.911000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>199</td>\n",
+       "      <td>0.941800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>0.913000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>201</td>\n",
+       "      <td>0.921800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>202</td>\n",
+       "      <td>0.921700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>203</td>\n",
+       "      <td>0.914500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>204</td>\n",
+       "      <td>0.910500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>205</td>\n",
+       "      <td>0.906600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>206</td>\n",
+       "      <td>0.915100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>207</td>\n",
+       "      <td>0.881600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>208</td>\n",
+       "      <td>0.884700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>209</td>\n",
+       "      <td>0.902900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>0.882600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>211</td>\n",
+       "      <td>0.891000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>212</td>\n",
+       "      <td>0.914400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>213</td>\n",
+       "      <td>0.930400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>214</td>\n",
+       "      <td>0.891100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>215</td>\n",
+       "      <td>0.859300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>216</td>\n",
+       "      <td>0.891800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>217</td>\n",
+       "      <td>0.873000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>218</td>\n",
+       "      <td>0.925900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>219</td>\n",
+       "      <td>0.905700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>0.921200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>221</td>\n",
+       "      <td>0.890200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>222</td>\n",
+       "      <td>0.915800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>223</td>\n",
+       "      <td>0.887300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>224</td>\n",
+       "      <td>0.898300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>225</td>\n",
+       "      <td>0.865600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>226</td>\n",
+       "      <td>0.873900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>227</td>\n",
+       "      <td>0.904800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>228</td>\n",
+       "      <td>0.917900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>229</td>\n",
+       "      <td>0.923400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>0.939700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>231</td>\n",
+       "      <td>0.913400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>232</td>\n",
+       "      <td>0.873100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>233</td>\n",
+       "      <td>0.896700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>234</td>\n",
+       "      <td>0.892100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>235</td>\n",
+       "      <td>0.902100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>236</td>\n",
+       "      <td>0.927200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>237</td>\n",
+       "      <td>0.912900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>238</td>\n",
+       "      <td>0.872900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>239</td>\n",
+       "      <td>0.904700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>0.879600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>241</td>\n",
+       "      <td>0.879800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>242</td>\n",
+       "      <td>0.908800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>243</td>\n",
+       "      <td>0.909800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>244</td>\n",
+       "      <td>0.838400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>245</td>\n",
+       "      <td>0.889200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>246</td>\n",
+       "      <td>0.912900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>247</td>\n",
+       "      <td>0.879700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>248</td>\n",
+       "      <td>0.910700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>249</td>\n",
+       "      <td>0.845400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>0.882200</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "trainer = transformers.Trainer(\n",
+    "    model = model,\n",
+    "    train_dataset = data,\n",
+    "    args = targs,\n",
+    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
+    ")\n",
+    "trainer.train(resume_from_checkpoint=False)\n",
+    "model.save_pretrained('sqllama-out2')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
+      "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "from model\n",
+      "<unk>table: 1-12028543-3\n",
+      "columns: Season,Cup FinalDate,WinningTeam,Score,LosingTeam,Location,Cup Final Attendance\n",
+      "Q: Who was the winning team in the 1989 season?\n",
+      "A:  SELECT  WinningTeam FROM 1-12028543-3 WHERE Season = '1989'\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  WinningTeam FROM 1-12028543-3 WHERE Season = '1989'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-18096431-5\n",
+      "columns: Place,Player,Country,Score,To par\n",
+      "Q: What is To par, when Country is \"United States\", and when Player is \"Mark Brooks\"?\n",
+      "A: 18-1\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  To par FROM 2-18096431-5 WHERE Country = 'united states' AND Player = 'mark brooks'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-10701914-2\n",
+      "columns: Home team,Home team score,Away team,Away team score,Venue,Crowd,Date\n",
+      "Q: What home team played at the western oval?\n",
+      "A:  Western Bulldogs\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Home team FROM 2-10701914-2 WHERE Venue = 'western oval'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-29598261-1\n",
+      "columns: Name,Number,Position,Height,Weight,Year,Hometown,Last School/College\n",
+      "Q: what is the school for chris mcnamara?\n",
+      "A:  SELECT  Last School/College FROM 1-29598261-1 WHERE Name = 'chris mcnamara'\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Last School/College FROM 1-29598261-1 WHERE Name = 'Chris McNamara'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-27722408-11\n",
+      "columns: Game,Date,Team,Score,High points,High rebounds,High assists,Location Attendance,Record\n",
+      "Q: Who had the most assists and how many did they have on April 8?\n",
+      "A:  SELECT  High assists FROM 1-27722408-11 WHERE Date = 'april 8'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  High assists FROM 1-27722408-11 WHERE Date = 'April 8'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-21378339-5\n",
+      "columns: Draw,Song,Artist,Panel Points,Televotes,Televote Points,Score,Placing\n",
+      "Q: Name the number of artists for panel points being 5\n",
+      "A:  SELECT COUNT Artist FROM 1-21378339-5 WHERE Panel Points = 5\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "expected answer\n",
+      "SELECT COUNT Artist FROM 1-21378339-5 WHERE Panel Points = 5\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-11545282-17\n",
+      "columns: Player,Nationality,Position,Years for Jazz,School/Club Team\n",
+      "Q: What position does Michael Ruffin play?\n",
+      "A:  SELECT  Position FROM 2-11545282-17 WHERE Player = 'michael ruffin'\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Position FROM 2-11545282-17 WHERE Player = 'michael ruffin'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-17801022-1\n",
+      "columns: Year,Date,Driver,Manufacturer,Laps,Miles (km),Race Time,Average Speed (mph)\n",
+      "Q: What manufacturer won the race on November 2?\n",
+      "A:  SELECT  Manufacturer FROM 1-17801022-1 WHERE Date = 'november 2'\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Manufacturer FROM 1-17801022-1 WHERE Date = 'November 2'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-10806592-14\n",
+      "columns: Home team,Home team score,Away team,Away team score,Venue,Crowd,Date\n",
+      "Q: What was the away score when the home team was Melbourne?\n",
+      "A:  SELECT  Away team score FROM 2-10806592-14 WHERE Home team = 'melbourne'\n",
+      "END\n",
+      "END\n",
+      "END\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Away team score FROM 2-10806592-14 WHERE Home team = 'melbourne'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-17978030-6\n",
+      "columns: Date,Time,Score,Set 1,Set 2,Set 3,Total\n",
+      "Q: What is the score when the set 3 is 26–28?\n",
+      "A:  SELECT  Score FROM 2-17978030-6 WHERE Set 3 = '26–28'\n",
+      "END\n",
+      "END\n",
+      "Q: What\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Score FROM 2-17978030-6 WHERE Set 3 = '26–28'\n",
+      "END\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_query(q):\n",
+    "    \n",
+    "    toks = tokenizer(q , return_tensors='pt')\n",
+    "    ctoks = toks.input_ids.to('cuda')\n",
+    "    gen = model.generate(ctoks, max_length=100)\n",
+    "    return tokenizer.decode(gen[0])\n",
+    "\n",
+    "M = len(q_red)\n",
+    "\n",
+    "for _ in range(10):\n",
+    "    j = random.randint(0,M-1)\n",
+    "    qs = q_red[j]\n",
+    "    a = a_red[j]\n",
+    "\n",
+    "    ma = get_query(qs)\n",
+    "\n",
+    "    #print(qs)\n",
+    "    print('from model')\n",
+    "    print(ma)\n",
+    "    print()\n",
+    "    print('expected answer')\n",
+    "    print(a)\n"
    ]
   }
  ],