KBR's Stacked Thoughts

1
2
3
!wget -nv -O gemini.pdf https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/2403.05530.pdf

document_file = client.files.upload(file='gemini.pdf')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
request = 'Tell me about the training process used here.'

def summarise_doc(request: str) -> str:
  """업로드된 문서에 대해 요청 실행하기"""
  # 출력 결과의 일관성을 높이기 위해 temp를 0.0으로 설정하기
  config = types.GenerateContentConfig(temperature=0.0)
  response = client.models.generate_content(
      model='gemini-2.0-flash',
      config=config,
      # 요청과 문서를 함께 전달
      contents=[request, document_file],
  )

  return response.text

summary = summarise_doc(request)
Markdown(summary)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import enum

# Define the evaluation prompt
SUMMARY_PROMPT = """\
# Instruction
You are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models.
We will provide you with the user input and an AI-generated responses.
You should first read the user input carefully for analyzing the task, and then evaluate the quality of the responses based on the Criteria provided in the Evaluation section below.
You will assign the response a rating following the Rating Rubric and Evaluation Steps. Give step-by-step explanations for your rating, and only choose ratings from the Rating Rubric.
Write your entire evaluation and explanation in Korean.

# Evaluation
## Metric Definition
You will be assessing summarization quality, which measures the overall ability to summarize text. Pay special attention to length constraints, such as in X words or in Y sentences. The instruction for performing a summarization task and the context to be summarized are provided in the user prompt. The response should be shorter than the text in the context. The response should not contain information that is not present in the context.

## Criteria
Instruction following: The response demonstrates a clear understanding of the summarization task instructions, satisfying all of the instruction's requirements.
Groundedness: The response contains information included only in the context. The response does not reference any outside information.
Conciseness: The response summarizes the relevant details in the original text without a significant loss in key information without being too verbose or terse.
Fluency: The response is well-organized and easy to read.

## Rating Rubric
5: (Very good). The summary follows instructions, is grounded, is concise, and fluent.
4: (Good). The summary follows instructions, is grounded, concise, and fluent.
3: (Ok). The summary mostly follows instructions, is grounded, but is not very concise and is not fluent.
2: (Bad). The summary is grounded, but does not follow the instructions.
1: (Very bad). The summary is not grounded.

## Evaluation Steps
STEP 1: Assess the response in aspects of instruction following, groundedness, conciseness, and verbosity according to the criteria.
STEP 2: Score based on the rubric.

# User Inputs and AI-generated Response
## User Inputs

### Prompt
{prompt}

## AI-generated Response
{response}
"""


# 평가 점수를 나타내는 enum 클래스
class SummaryRating(enum.Enum):
  VERY_GOOD = '5'
  GOOD = '4'
  OK = '3'
  BAD = '2'
  VERY_BAD = '1'

def eval_summary(prompt, ai_response):
  """주어진 프롬프트에 대해 생성된 요약문을 평가"""

  # Gemini 모델을 사용하여 채팅 세션 생성
  chat = client.chats.create(model='gemini-2.0-flash')

  # 평가 프롬프트를 문자열 formatiing으로 완성하고, 평가를 요청한다. 
  response = chat.send_message(
      message=SUMMARY_PROMPT.format(prompt=prompt, response=ai_response)
  )
  verbose_eval = response.text

  # 점수를 enum 형태로 변환하기 위한 설정 구성
  structured_output_config = types.GenerateContentConfig(
      response_mime_type="text/x.enum",
      response_schema=SummaryRating,
  )
  
  # 최종 점수만 추출해서 다시 요청
  response = chat.send_message(
      message="Convert the final score.",
      config=structured_output_config,
  )
  # 정형화된 점수 결과
  structured_eval = response.parsed

  return verbose_eval, structured_eval

# 평가 실행: 자세한 설명과 점수 받기
text_eval, struct_eval = eval_summary(prompt=[request, document_file], ai_response=summary)
# 마크다운 형식으로 출력
Markdown(text_eval)

1
print(struct_eval)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# new prompt 설정: 훈련 과정을 5살 어린이에게 말하듯이 설명해줘
new_prompt = "Explain in Korean like I'm 5 the training process"
# Try:
#  ELI5 the training process
#  Summarise the needle/haystack evaluation technique in 1 line
#  Describe the model architecture to someone with a civil engineering degree
#  What is the best LLM?

# prompt가 비어있는 경우 에러 일으키기
if not new_prompt:
  raise ValueError("Try setting a new summarisation prompt.")

# 새로운 프롬프트로 요약문을 만들고, 그 결과를 평가하기
def run_and_eval_summary(prompt):
  # `summary_doc(new_prompt)`을 호출해 요약문 생성
  summary = summarise_doc(new_prompt)
  # 요약 결과 출력
  display(Markdown(summary + '\n-----'))
  # `eval_summary` 함수로 요약 결과 평가하기
  text, struct = eval_summary([new_prompt, document_file], summary)
  display(Markdown(text + '\n-----'))
  print(struct)

run_and_eval_summary(new_prompt)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# new prompt 설정: 훈련 과정을 토목 공학 전공자에게 말하듯이 설명해줘
new_prompt = "Describe the model architecture to someone with a civil engineering degree, in Korean"

# prompt가 비어있는 경우 에러 일으키기
if not new_prompt:
  raise ValueError("Try setting a new summarisation prompt.")

# 새로운 프롬프트로 요약문을 만들고, 그 결과를 평가하기
def run_and_eval_summary(prompt):
  # `summary_doc(new_prompt)`을 호출해 요약문 생성
  summary = summarise_doc(new_prompt)
  # 요약 결과 출력
  display(Markdown(summary + '\n-----'))
  # `eval_summary` 함수로 요약 결과 평가하기
  text, struct = eval_summary([new_prompt, document_file], summary)
  display(Markdown(text + '\n-----'))
  print(struct)

run_and_eval_summary(new_prompt)