Skip to content
Snippets Groups Projects
Commit 8ae8d01d authored by Michael Youkeim's avatar Michael Youkeim
Browse files

Protobuf works now

parent a3d34557
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Dockerfile
+ 9
− 8
View file @ 8ae8d01d
......@@ -52,14 +52,15 @@ RUN curl -L https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz > cmake-3.19.0.tar
make -j$(nproc) && \
make install
# Install Protocol Buffers 3.21.12
RUN curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz && \
tar -zxvf protobuf-3.21.12.tar.gz && \
cd protobuf-3.21.12 && \
./autogen.sh && \
./configure && \
make -j$(nproc) && \
make install
# Download and install Protocol Buffers 2.5.0
RUN wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
&& tar xvf protobuf-2.5.0.tar.gz \
&& cd protobuf-2.5.0 \
&& ./autogen.sh \
&& ./configure --prefix=/usr \
&& make \
&& make install \
&& ldconfig
# Set LD_LIBRARY_PATH to include /usr/local/lib and run ldconfig
ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
......
src/hadoop_analysis/build_manager.py
+ 8
− 183
View file @ 8ae8d01d
......@@ -8,106 +8,6 @@ import json
from java_version_manager import switch_java_version
def create_maven_settings(repo_path):
"""Create or modify Maven settings to enforce HTTPS."""
settings_path = os.path.join(repo_path, "settings.xml")
settings_content = f"""
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
https://maven.apache.org/xsd/settings-1.0.0.xsd">
<mirrors>
<mirror>
<id>central-https</id>
<mirrorOf>central</mirrorOf>
<name>Maven Central mirror over HTTPS</name>
<url>https://repo.maven.apache.org/maven2</url>
</mirror>
</mirrors>
</settings>
"""
with open(settings_path, "w") as file:
file.write(settings_content)
return settings_path
def parse_pom_xml(file_path):
tree = ET.parse(file_path)
root = tree.getroot()
ns = {"maven": "http://maven.apache.org/POM/4.0.0"}
enforcer_plugin = root.find(
".//maven:plugin[maven:artifactId='maven-enforcer-plugin']", ns
)
java_version = None
maven_version = None
if enforcer_plugin:
java_version = enforcer_plugin.find(
".//maven:requireJavaVersion/maven:version", ns
)
maven_version = enforcer_plugin.find(
".//maven:requireMavenVersion/maven:version", ns
)
java_version = java_version.text if java_version is not None else None
maven_version = maven_version.text if maven_version is not None else None
if not java_version:
java_version = root.find(
".//plugin[artifactId='maven-enforcer-plugin']//requireJavaVersion/version"
)
maven_version = root.find(
".//plugin[artifactId='maven-enforcer-plugin']//requireMavenVersion/version"
)
java_version = java_version.text if java_version is not None else None
maven_version = maven_version.text if maven_version is not None else None
return java_version, maven_version
def get_installed_version(command):
try:
result = subprocess.run(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
shell=True,
)
return result.stdout.strip()
except Exception as e:
print("Error obtaining installed version:", e)
return None
def normalize_version(ver):
ver = ver.replace("_", ".")
match = re.match(r"(\d+\.\d+\.\d+)", ver)
return match.group(1) if match else ver
def version_is_compatible(required_version_range, installed_version):
if required_version_range and installed_version:
try:
normalized_installed_version = normalize_version(installed_version)
if "," in required_version_range:
lower_bound = required_version_range.strip("[]()").split(",")[0].strip()
if version.parse(normalized_installed_version) >= version.parse(
lower_bound
):
return True
else:
required_version = required_version_range.strip("[]()")
required_ver = version.parse(required_version.split("_")[0])
installed_ver = version.parse(installed_version.split("_")[0])
return (required_ver.major == installed_ver.major) and (
required_ver.minor == installed_ver.minor
)
except version.InvalidVersion as e:
print(f"Error parsing version: {e}")
return False
def copy_jar_files(source_dir, output_dir):
# if os.path.exists(output_dir):
......@@ -140,92 +40,17 @@ def process_commit_hashes(commit_hashes, repo_path, output_dir):
subprocess.run(["git", "checkout", hash], cwd=repo_path, check=True)
print(f"Checked out {hash}")
# # Run Maven to build the project for each commit
# subprocess.run(
# ["mvn", "clean", "install", "-DskipTests", "--projects", ":hadoop-common", "--also-make"],
# cwd=repo_path,
# check=True
# )
# Get repo requirements
java_required, maven_required = parse_pom_xml(
os.path.join(repo_path, "pom.xml")
)
norm_java_required = (
(normalize_version(java_required).split("_")[0])
.replace("[", "")
.replace(")", "")
.replace(",", "")
)
norm_maven_required = (
(normalize_version(maven_required).split("_")[0])
.replace("[", "")
.replace(")", "")
.replace(",", "")
)
print("=" * 10)
print(f"Required Java Version : {norm_java_required}")
print(f"Required Maven Version : {norm_maven_required}\n")
java_versions[hash] = {
"java": norm_java_required,
"maven": norm_maven_required,
}
switch_java_version(norm_java_required)
java_installed = get_installed_version(
"java -version 2>&1 | head -n 1 | awk '{print $3}' | tr -d '\"'"
)
maven_installed = get_installed_version(
"mvn -v | grep 'Apache Maven' | awk '{print $3}'"
)
print(
f"Installed Java Version : {normalize_version(java_installed).split('_')[0]}"
)
print(
f"Installed Maven Version : {normalize_version(maven_installed).split('_')[0]}"
)
print("=" * 10)
is_version_compatible = version_is_compatible(java_required, java_installed)
java_versions[hash]["compatible"] = is_version_compatible
is_version_compatible = (
True # Temporarily skipping checking compatible versions
)
# Generate sources
# subprocess.run(f"cd {repo_path} && mvn clean install -DskipTests -DskipShade", shell=True, check=True)
# Generate sources
# subprocess.run(f"cd {repo_path} && mvn generate-sources", shell=True, check=True)
# Proceed if versions are compatible
if (
is_version_compatible
): # and version_is_compatible(maven_required, maven_installed):
# Run Maven to build the project
settings_path = create_maven_settings(repo_path)
maven_command = (
f"cd {repo_path} && mvn --settings {settings_path} clean package "
f"-Dhttps.protocols=TLSv1.2 --projects :hadoop-common --also-make "
f"-DskipTests -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true "
f"-Dcommon.protobuf2.scope=compile"
)
maven_command = f"cd {repo_path} && mvn clean package -U --projects :hadoop-common --also-make -DskipTests"
subprocess.run(maven_command, shell=True, check=True)
# Copy the jar files to the specified output directory
copy_jar_files(
f"{repo_path}/hadoop-common-project/hadoop-common/target",
f"{output_dir}/{hash}",
)
else:
print(f"Incompatible java versions. Aborting ...\n\n")
except subprocess.CalledProcessError as e:
print(
f"Error: {e}. Command '{e.cmd}' returned non-zero exit status {e.returncode}."
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment