def tgi_server(prompt):
headers = {'Content-Type': 'application/json'}
url = f'.../generate'
data = {
"inputs": prompt,
"parameters": {
"max_new_tokens": 1000,
"temperature": 1.0,
"top_p": 0.99,
"do_sample":False,
"seed": 42
}
}
response = requests.post(url, json=data, headers=headers)
# print(response.status_code)
res = response.json()
# print(res)
# print(response.status_code)
return res
if __name__ == '__main__':
for index, sample in enumerate(input_sample_data):
input_text = '...'
input_str = f'"""{input_text}"""'
template = f"""[INST] <<SYS>> ...
<</SYS>>
{input_str}[/INST]"""
print("starting on {}".format(InsightSourceId))
s0 = time()
# print(template)
response = tgi_server(template)
s1 = time()
# print(response)
response = response["generated_text"]
def tgi_server(prompt):
headers = {'Content-Type': 'application/json'}
url = f'.../generate'
data = {
"inputs": prompt,
"parameters": {
"max_new_tokens": 1000,
"temperature": 1.0,
"top_p": 0.99,
"do_sample":False,
"seed": 42
}
}
response = requests.post(url, json=data, headers=headers)
# print(response.status_code)
res = response.json()
# print(res)
# print(response.status_code)
return res
if __name__ == '__main__':
for index, sample in enumerate(input_sample_data):
input_text = '...'
input_str = f'"""{input_text}"""'
template = f"""[INST] <<SYS>> ...
<</SYS>>
{input_str}[/INST]"""
print("starting on {}".format(InsightSourceId))
s0 = time()
# print(template)
response = tgi_server(template)
s1 = time()
# print(response)
response = response["generated_text"]