AI level 3, task 1, grader fails with with "No worries!" (general test case failure)

Artificial Intelligence
#1

The grader appears to be running the wrong working directory, so I have added some workarounds (os.path.join…) to concatenate path to the files used in the task. My suggestion looks like this:

from openai import OpenAI
import os
import json
import logging

client = OpenAI(
    api_key = os.environ.get('OPENAI_API_KEY'),
    base_url = os.environ.get('OPENAI_API_BASE'),
)
BASE_DIR = os.path.dirname(os.path.abspath(__file__))

def generate_code(requirements_path: str, output_path: str):
    with open(os.path.join(BASE_DIR, requirements_path), 'r', encoding='utf-8') as file:
        reqs = file.read()
    prompt = f"""
    Based on these requirements, write a Python script that scrapes all <h2> headlines from 'example.com' and prints them. Include imports and a main execution block. Respond ONLY with the raw Python code.
    Do not add Markdown formatting to the response.
    Requirements: {reqs}
    """
    response = client.chat.completions.create(
        model="openai/gpt-4.1-mini",
        messages=[
            {"role": "user", "content": prompt}
        ],
        max_tokens=120,
        temperature=0.0,
    )
    code = response.choices[0].message.content
    with open(os.path.join(BASE_DIR, output_path), "w", encoding="utf-8") as file:
        file.write(code)

def review_code(code_path: str) -> str:
    with open(os.path.join(BASE_DIR, code_path), 'r', encoding='utf-8') as file:
        code = file.read()
    prompt = f"""
    You are a senior security architect. Review the attached Python code for any security or best-practice vulnerabilities (e.g., missing 'User-Agent' headers, no error handling for HTTP 4xx/5xx requests). Output a JSON object with a single key 'findings', which is a list of strings, each string describing one vulnerability. If no issues, return an empty list.
    Respond in pure JSON format, do not add Markdown formatting.
    """
    response = client.chat.completions.create(
        model="openai/gpt-4.1-mini",
        messages=[
            {"role": "system", "content": prompt},
            {"role": "user", "content": code}
        ],
        max_tokens=120,
        temperature=0.5,
    )
    return response.choices[0].message.content

if __name__ == "__main__":
    logging.basicConfig(
       filename='/root/openaiproject/output.log',         # Name of the file
       filemode='w',                # 'a' to append logs, 'w' to overwrite each run
        format='%(asctime)s - %(levelname)s - %(message)s',
        level=logging.INFO           # Minimum severity level to capture
    )
    try:
        generate_code('requirements.txt', 'web_scraper.py')
        response = review_code('web_scraper.py')
        print(response)
    except Exception as e:
        logging.exception("An unhandled exception occurred in main:")

which runs fine:

~/openaiproject ➜  python pipeline.py 
{
  "findings": [
    "The HTTP request does not include a 'User-Agent' header, which can lead to the request being blocked or treated as suspicious by some servers.",
    "There is no error handling for HTTP errors beyond raise_for_status(), which will raise an exception but does not allow for graceful recovery or logging of specific error details.",
    "The code does not handle potential exceptions from network issues such as timeouts or connection errors.",
    "The URL used is HTTP rather than HTTPS, which means the data is transmitted in plaintext and could be intercepted or tampered with."

~/openaiproject ➜

and the output is valid JSON.

The grader output is this:

============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-9.0.3, pluggy-1.6.0
rootdir: /usr/share
plugins: testinfra-10.2.2, anyio-4.13.0
collected 1 item

../usr/share/test.py F                                                   [100%]

=================================== FAILURES ===================================
________________________ test_security_pipeline_script _________________________

    def test_security_pipeline_script():
        """
        Runs pipeline.py and checks that it:
        1. Creates the 'web_scraper.py' file.
        2. Prints a valid JSON string containing the 'findings' key.
        """

        # 1. Check if the main script exists
        assert os.path.exists(FILE_PATH), "'pipeline.py' file does not exist."

        # Read the content of pipeline.py before using the 'content' variable
        with open(FILE_PATH, "r") as f:
            content = f.read()

        # 2. Check api_key and base_url (hardcoded or env-based)
        has_explicit_api_and_base = "api_key" in content and "base_url" in content
        has_env_indicators = (
            ("os.environ" in content or "os.getenv" in content or "os.environ.get" in content)
            and (
                (("API_KEY" in content) or ("OPENAI_API_KEY" in content))
                and (("BASE_URL" in content) or ("OPENAI_API_BASE" in content))
            )
        )
        assert has_explicit_api_and_base or has_env_indicators, \
            "OpenAI client not initialized with api_key/base_url literals or environment-derived values"

        # 3. Check for key components in the script file
        assert "openai/gpt-4.1-mini" in content, \
            "The correct OpenAI model is not used."
        assert "temperature=0.0" in content, \
            "The 'temperature' parameter must be set to 0.0 for Call 1."
        assert "temperature=0.5" in content, \
            "The 'temperature' parameter must be set to 0.5 for Call 2."
        assert "def generate_code" in content, \
            "Function 1 is not defined correctly."
        assert "def review_code" in content, \
            "Function 2 is not defined correctly."
        assert "web_scraper.py" in content, \
            "The script must write to 'web_scraper.py' file."

        # 4. Clean up any previous runs
        if os.path.exists(GENERATED_CODE_PATH):
            os.remove(GENERATED_CODE_PATH)

        # 5. Run the user's script
        command = (
            "/bin/bash -c 'source /root/openaiproject/venv/bin/activate && "
            "{ [ ! -f /root/.bash_profile ] || { set -a; . /root/.bash_profile; set +a; }; } && "
            "python3 /root/openaiproject/pipeline.py'"
        )
        result = subprocess.run(
            command,
            shell=True,
            capture_output=True,
            text=True,
            timeout=60
        )

        # 6. Verify program execution
        assert result.returncode == 0, (
            f"'pipeline.py' exited with a non-zero status code: "
            f"{result.returncode}\nStderr: {result.stderr}"
        )

        # 7. Check File I/O (Test 1: Did it create the file?)
        assert os.path.exists(GENERATED_CODE_PATH), (
            "The 'web_scraper.py' file was not created by 'generate_code' function 1."
        )

        # 8. Verify the generated file contains Python code
        with open(GENERATED_CODE_PATH, "r") as f:
            generated_content = f.read()

        assert (
            "def " in generated_content
            and "import " in generated_content
        ), "The generated file does not appear to contain valid Python code."

        # 9. Check Final Output (Test 2: Is it valid JSON?)
        output = result.stdout.strip()
        assert output, "pipeline.py produced no output."

        # Extract only the last non-empty line, which should be the JSON output
        output_lines = [line.strip() for line in output.splitlines() if line.strip()]
        json_output_string = output_lines[-1]

        # Parse the JSON output
        try:
>           review_json = json.loads(json_output_string)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

/usr/share/test.py:100: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python3.12/json/__init__.py:346: in loads
    return _default_decoder.decode(s)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
/usr/lib/python3.12/json/decoder.py:337: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7842f9816fc0>, s = '}', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.

        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.

        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/usr/lib/python3.12/json/decoder.py:355: JSONDecodeError

During handling of the above exception, another exception occurred:

    def test_security_pipeline_script():
        """
        Runs pipeline.py and checks that it:
        1. Creates the 'web_scraper.py' file.
        2. Prints a valid JSON string containing the 'findings' key.
        """

        # 1. Check if the main script exists
        assert os.path.exists(FILE_PATH), "'pipeline.py' file does not exist."

        # Read the content of pipeline.py before using the 'content' variable
        with open(FILE_PATH, "r") as f:
            content = f.read()

        # 2. Check api_key and base_url (hardcoded or env-based)
        has_explicit_api_and_base = "api_key" in content and "base_url" in content
        has_env_indicators = (
            ("os.environ" in content or "os.getenv" in content or "os.environ.get" in content)
            and (
                (("API_KEY" in content) or ("OPENAI_API_KEY" in content))
                and (("BASE_URL" in content) or ("OPENAI_API_BASE" in content))
            )
        )
        assert has_explicit_api_and_base or has_env_indicators, \
            "OpenAI client not initialized with api_key/base_url literals or environment-derived values"

        # 3. Check for key components in the script file
        assert "openai/gpt-4.1-mini" in content, \
            "The correct OpenAI model is not used."
        assert "temperature=0.0" in content, \
            "The 'temperature' parameter must be set to 0.0 for Call 1."
        assert "temperature=0.5" in content, \
            "The 'temperature' parameter must be set to 0.5 for Call 2."
        assert "def generate_code" in content, \
            "Function 1 is not defined correctly."
        assert "def review_code" in content, \
            "Function 2 is not defined correctly."
        assert "web_scraper.py" in content, \
            "The script must write to 'web_scraper.py' file."

        # 4. Clean up any previous runs
        if os.path.exists(GENERATED_CODE_PATH):
            os.remove(GENERATED_CODE_PATH)

        # 5. Run the user's script
        command = (
            "/bin/bash -c 'source /root/openaiproject/venv/bin/activate && "
            "{ [ ! -f /root/.bash_profile ] || { set -a; . /root/.bash_profile; set +a; }; } && "
            "python3 /root/openaiproject/pipeline.py'"
        )
        result = subprocess.run(
            command,
            shell=True,
            capture_output=True,
            text=True,
            timeout=60
        )

        # 6. Verify program execution
        assert result.returncode == 0, (
            f"'pipeline.py' exited with a non-zero status code: "
            f"{result.returncode}\nStderr: {result.stderr}"
        )

        # 7. Check File I/O (Test 1: Did it create the file?)
        assert os.path.exists(GENERATED_CODE_PATH), (
            "The 'web_scraper.py' file was not created by 'generate_code' function 1."
        )

        # 8. Verify the generated file contains Python code
        with open(GENERATED_CODE_PATH, "r") as f:
            generated_content = f.read()

        assert (
            "def " in generated_content
            and "import " in generated_content
        ), "The generated file does not appear to contain valid Python code."

        # 9. Check Final Output (Test 2: Is it valid JSON?)
        output = result.stdout.strip()
        assert output, "pipeline.py produced no output."

        # Extract only the last non-empty line, which should be the JSON output
        output_lines = [line.strip() for line in output.splitlines() if line.strip()]
        json_output_string = output_lines[-1]

        # Parse the JSON output
        try:
            review_json = json.loads(json_output_string)
        except json.JSONDecodeError:
>           pytest.fail(
                "Final output is not valid JSON.\n"
                f"Text Found: {json_output_string}"
            )
E           Failed: Final output is not valid JSON.
E           Text Found: }

/usr/share/test.py:102: Failed
=========================== short test summary info ============================
FAILED ../usr/share/test.py::test_security_pipeline_script - Failed: Final ou...
============================== 1 failed in 4.11s ===============================
#2

Our engineers would likely be dismayed that you hacked our grader :slight_smile: :joy_cat: Kudos!

I’ve opened a ticket on this lab on behalf of another student; the grader is clearly failing folks who also got reasonable output from the lab, and my experience with this lab is similar to the two of you. We’ll see what the lab team has to say in reply :wink:

#3

OK, well done – the engineer came to exactly your conclusion. Your logging code also annoys the grader (which he didn’t allow for), but if you remove your logging code, I think your solution will now work fine.

#4

Hmm, I didn’t try to hack the grader in any way. The reason for adding the logging code, was that I originally just got “Process exited with status code 1” or similar as error. The logging indicated that the grader looked up files in the wrong directory. Working around that, I got the above error.

However, I still get the same error after removing the logging code (keeping the path adjustments, though).

#5

This is my last attempt, with no logging code and no path adjustmet workaround:

from openai import OpenAI
import os
import json

client = OpenAI(
    api_key = os.environ.get('OPENAI_API_KEY'),
    base_url = os.environ.get('OPENAI_API_BASE'),
)
BASE_DIR = os.path.dirname(os.path.abspath(__file__))

def generate_code(requirements_path: str, output_path: str):
    with open(requirements_path, 'r', encoding='utf-8') as file:
        reqs = file.read()
    prompt = f"""
    Based on these requirements, write a Python script that scrapes all <h2> headlines from 'example.com' and prints them. Include imports and a main execution block. Respond ONLY with the raw Python code.
    Do not add Markdown formatting to the response.
    Requirements: {reqs}
    """
    response = client.chat.completions.create(
        model="openai/gpt-4.1-mini",
        messages=[
            {"role": "user", "content": prompt}
        ],
        max_tokens=120,
        temperature=0.0,
    )
    code = response.choices[0].message.content
    with open(output_path, "w", encoding="utf-8") as file:
        file.write(code)

def review_code(code_path: str) -> str:
    with open(code_path, 'r', encoding='utf-8') as file:
        code = file.read()
    prompt = f"""
    You are a senior security architect. Review the attached Python code for any security or best-practice vulnerabilities (e.g., missing 'User-Agent' headers, no error handling for HTTP 4xx/5xx requests). Output a JSON object with a single key 'findings', which is a list of strings, each string describing one vulnerability. If no issues, return an empty list.
    Respond in pure JSON format, do not add Markdown formatting.
    """
    response = client.chat.completions.create(
        model="openai/gpt-4.1-mini",
        messages=[
            {"role": "system", "content": prompt},
            {"role": "user", "content": code}
        ],
        max_tokens=120,
        temperature=0.5,
    )
    return response.choices[0].message.content

generate_code('requirements.txt', 'web_scraper.py')
response = review_code('web_scraper.py')
print(response)

which runs and generates:

~/openaiproject ➜  python pipeline.py 
{
  "findings": [
    "The HTTP request does not include a 'User-Agent' header, which can lead to blocking by some servers or misidentification of the client.",
    "There is no error handling for HTTP 4xx or 5xx responses other than raise_for_status(), which will raise an exception and terminate the program without graceful recovery or logging.",
    "The URL uses 'http' instead of 'https', which is not secure and could lead to man-in-the-middle attacks."
  ]
}
#6

No judging here – just admiring your ingenuity.

#7

Ok, I finished the task. I changed the second function, review_code, to use the OpenAI APIs
structured outputs: Structured model outputs | OpenAI API, with a Pydantic object representing the structure of the JSON output. I suspect there is an issue here of non-deterministic output (sometimes JSON, sometimes not), confusing the grader, if you don’t force the output to be JSON with structured outputs.