Skip to content
Snippets Groups Projects
Commit 90ed6b56 authored by Michael Youkeim's avatar Michael Youkeim
Browse files

Added newer image

parent a76cd825
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Dockerfile
+ 41
− 20
View file @ 90ed6b56
......@@ -5,21 +5,29 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \
wget \
curl \
gnupg \
protobuf-compiler \
git \
maven \
software-properties-common \
python3 \
maven \
python3-pip \
autoconf \
automake \
libtool \
g++ \
make \
unzip \
pkg-config \
libcairo2 \
libcairo2-dev
libcairo2-dev \
build-essential \
zlib1g-dev \
libssl-dev \
libsasl2-dev
# Install GCC 9
RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
apt-get update && \
apt-get install -y gcc-9 g++-9 && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9
# Add the Temurin repository and install Java
RUN wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor > /usr/share/keyrings/temurin-archive-keyring.gpg \
......@@ -36,31 +44,44 @@ RUN tar -xzf /tmp/jdk-6u45-linux-x64.tar.gz -C /usr/lib/jvm/ && \
update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.6.0_45/bin/java 100 && \
update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-se-7u75-ri/bin/java 200
# Download and install Protocol Buffers 2.5.0
RUN wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
&& tar xvf protobuf-2.5.0.tar.gz \
&& cd protobuf-2.5.0 \
&& ./autogen.sh \
&& ./configure --prefix=/usr \
&& make \
&& make install \
&& ldconfig /usr/local/lib \
&& cd ..
# Install CMake 3.19
RUN curl -L https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz > cmake-3.19.0.tar.gz && \
tar -zxvf cmake-3.19.0.tar.gz && \
cd cmake-3.19.0 && \
./bootstrap && \
make -j$(nproc) && \
make install
# Install Protocol Buffers 3.21.12
RUN curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz && \
tar -zxvf protobuf-3.21.12.tar.gz && \
cd protobuf-3.21.12 && \
./autogen.sh && \
./configure && \
make -j$(nproc) && \
make install
# Set LD_LIBRARY_PATH to include /usr/local/lib and run ldconfig
ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
RUN ldconfig
# Install Boost Libraries
# RUN curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 && \
# tar --bzip2 -xf boost_1_72_0.tar.bz2 && \
# cd boost_1_72_0 && \
# ./bootstrap.sh --prefix=/usr/ && \
# ./b2 --without-python && \
# ./b2 --without-python install
# Cleanup
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Set the default Java
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
ENV PATH $JAVA_HOME/bin:$PATH
# Set the working directory and copy your app
WORKDIR /app
COPY . .
# Install Python dependencies
RUN pip3 install -r requirements.txt #solve pip issue
#RUN pip3 install -r requirements.txt
RUN pip3 install -r requirements.txt
# Keep the container running
CMD ["tail", "-f", "/dev/null"]
data/output/java_versions.json
+ 416
− 661
View file @ 90ed6b56
......@@ -54,21 +54,6 @@
"maven": "3.0.2",
"compatible": false
},
"d44513c3767bc0c6490273fc486c1c26d90ab3f4": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"8ad1fcb3c3e27c91935a4ef55a92d0bf244f8d08": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"743cbf6ef3f64593c141e7dd302ad2a2d2e3b197": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"8a9e63e468155f2a57bf19f82f920a7cc9d1d302": {
"java": "1.6",
"maven": "3.0.2",
......@@ -84,11 +69,6 @@
"maven": "3.0.2",
"compatible": false
},
"3dd40a70ca7861d349f5750a2f865ffdfb464aaa": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"4bca22005f48f426b9bc7cf36d435ead470a2590": {
"java": "1.6",
"maven": "3.0.2",
......@@ -114,11 +94,6 @@
"maven": "3.0.2",
"compatible": false
},
"a4f30ff53c10672ec9ad847b4892f39e94fb1bf8": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"70b1c5ca7562d2b21450bb80858f261092304597": {
"java": "1.6",
"maven": "3.0.2",
......@@ -144,16 +119,6 @@
"maven": "3.0.2",
"compatible": false
},
"8962f5ce88bba03d113c5849db8c464bfd1669a8": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"cda2564e1c7b37ba32dec9c54b9de3fbfb82da8b": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"8ef140d38b16a18050e9e7901e8aadd424aef20f": {
"java": "1.6",
"maven": "3.0.2",
......@@ -164,36 +129,16 @@
"maven": "3.0.2",
"compatible": false
},
"cdfc555eae5066d88075dbfe6e3379d9a54e4294": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"5f9b4c14a175873b4f82654513e289c657c694eb": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"b3a8d99817dcceb4d1125dec0c3ecbb0f15f6c76": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"febc5040b9aad24a6c9df9e7fe138965cf1c994b": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"569a9bd5f95085609894c1f07b75387c78659d6a": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"52f0259502de42e433588c299339bf5cd4ba1f8e": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"4402e2fceff56f2dbbf58df06a421c52344cfdc4": {
"java": "1.6",
"maven": "3.0.2",
......@@ -204,11 +149,6 @@
"maven": "3.0.2",
"compatible": false
},
"990e4945562974a285f31d8385230b03a9d557e7": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"e05ff82bd95dd802233e2424e8eafbd7ec6944c5": {
"java": "1.6",
"maven": "3.0.2",
......@@ -219,11 +159,6 @@
"maven": "3.0.2",
"compatible": false
},
"15b9077c2d41f59c716582d8f7ae6e334630c0ac": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"b20180ffa6c89396d9fcfec8b029b9c600503c3d": {
"java": "1.6",
"maven": "3.0.2",
......@@ -244,11 +179,6 @@
"maven": "3.0.2",
"compatible": false
},
"03c858dad46e440028c58a716630160338db4b14": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"ed642b5d8dd4cbe19518f0d36dc63700bc1bd7a9": {
"java": "1.6",
"maven": "3.0.2",
......@@ -284,36 +214,6 @@
"maven": "3.0.2",
"compatible": false
},
"d7200a4b59bac51076432634548c9b5a91e48f17": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"b0f0ef17f07c0140858bcb680f86fbbab3beccb8": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"d9a7404c389ea1adffe9c13f7178b54678577b56": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"2ca93d1fbf0fdcd6b4b5a151261052ac106ac9e1": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"59384dfb710f42d2a419c1b7db5a1a62a39be5f3": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"ac35e3464ff1b5bacf416fb78112681f0a9e7ed8": {
"java": "1.6",
"maven": "3.0.2",
"compatible": false
},
"24d920b80eb3626073925a1d0b6dcf148add8cc0": {
"java": "1.6",
"maven": "3.0.2",
......@@ -324,11 +224,6 @@
"maven": "3.0.2",
"compatible": true
},
"4ad484883f773c702a1874fc12816ef1a4a54136": {
"java": "1.7",
"maven": "3.0.2",
"compatible": true
},
"b1ed28fa77cb2fab80c54f9dfeb5d8b7139eca34": {
"java": "1.7",
"maven": "3.0.2",
......@@ -364,21 +259,11 @@
"maven": "3.0.2",
"compatible": true
},
"b9e3eff62a7415d8666656a75db69ff3e43f8e7e": {
"java": "1.7",
"maven": "3.0.2",
"compatible": true
},
"4838b735f0d472765f402fe6b1c8b6ce85b9fbf1": {
"java": "1.7",
"maven": "3.0.2",
"compatible": true
},
"d6402fadedade4289949ba9f70f7a0bfb9bca140": {
"java": "1.7",
"maven": "3.0.2",
"compatible": true
},
"2753185a010e70f8d9539f42151c79177781122d": {
"java": "1.7",
"maven": "3.0.2",
......@@ -409,66 +294,11 @@
"maven": "3.0.2",
"compatible": true
},
"60f66a930626255d60dea7acc828975061ec37de": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"dbb28eb1f2125b4a2e43977d9add80a45f359c28": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"7d4431c93bf4faaa8ca54e2f9a219bf7e1d506b7": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"005f4528c7b0aa4a48cf62b11e837be886f267d3": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"a49b3be38ed97a27f215afb996c6db516f5857d7": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"846c5ceb3a929ad7b2dcea9bef07299af17bdc84": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"5707f88d8550346f167e45c2f8c4161eb3957e3a": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"d1443988f809fe6656f60dfed4ee4e0f4844ee5c": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"a57bba470b396c163baef7ac9447c063180ec15b": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"669d6f13ec48a90d4ba7e4ed1dd0e9687580f8f3": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"6c348c56918973fd988b110e79231324a8befe12": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"4bca385241c0fc8ff168c7b0f2984a7aed2c7492": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"5c61ad24887f76dfc5a5935b2c5dceb6bfd99417": {
"java": "1.8",
"maven": "3.0.2",
......@@ -484,16 +314,6 @@
"maven": "3.0.2",
"compatible": true
},
"0665c5f09afb2d6f59b6c4d4980f8b1ff9cbe620": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"e648b6e1382336af69434dfbf9161bced3caa244": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"0c01cf57987bcc7a17154a3538960b67f625a9e5": {
"java": "1.8",
"maven": "3.0.2",
......@@ -509,11 +329,6 @@
"maven": "3.0.2",
"compatible": true
},
"144f1cf76527e6c75aec77ef683a898580f3cc8d": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"bf3fb585aaf2b179836e139c041fc87920a3c886": {
"java": "1.8",
"maven": "3.0.2",
......@@ -524,41 +339,21 @@
"maven": "3.0.2",
"compatible": true
},
"d670c3a4da7dd80dccf6c6308603bb3bb013b3b0": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"2843c688bcc21c65eb3538ffb3caeaffe440eda8": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"778d4edd9adbe9519c3d6df65e45ddc8bb0ab2da": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"4ec5acc70418a3f2327cf83ecae1789a057fdd99": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"621b43e254afaff708cd6fc4698b29628f6abc33": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"13eda5000304099d1145631f9be13ce8a00b600d": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"f19638333b11da6dcab9a964e73a49947b8390fd": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"9195a6e302028ed3921d1016ac2fa5754f06ebf0": {
"java": "1.8",
"maven": "3.0.2",
......@@ -569,36 +364,11 @@
"maven": "3.0.2",
"compatible": true
},
"b1e2704a3a7e64661085343ba0e8457cb4c9adcc": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"f044deedbbfee0812316d587139cb828f27172e9": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"1cf38a38da060ed06377099057a31f17d570c3c1": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"0def61482b7636161b0edcfee377c74f398953d3": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"046b8768af8a07a9e10ce43f538d6ac16e7fa5f3": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"4f752d442b437b3d297232cfdc8b04aa55c53c5e": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"1a25bbe9ec7155712a82b157a8379826f8c79d4b": {
"java": "1.8",
"maven": "3.0.2",
......@@ -639,24 +409,9 @@
"maven": "3.0.2",
"compatible": true
},
"93ad7c32f428b48d3499dfb3aa15249e8c5cc37d": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"f813554769606d59c23bcdc184d52249793d0f12": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"8d6a6869531abb3cda974358509f63ae61986a5c": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
},
"cb8c98fbb02847916e23ffaa60b0b54694d056d7": {
"java": "1.8",
"maven": "3.0.2",
"compatible": true
}
}
\ No newline at end of file
src/hadoop_analysis/build_manager.py
+ 25
− 15
View file @ 90ed6b56
......@@ -92,8 +92,7 @@ def version_is_compatible(required_version_range, installed_version):
try:
normalized_installed_version = normalize_version(installed_version)
if "," in required_version_range:
lower_bound = required_version_range.strip("[]()").split(",")[
0].strip()
lower_bound = required_version_range.strip("[]()").split(",")[0].strip()
if version.parse(normalized_installed_version) >= version.parse(
lower_bound
):
......@@ -134,15 +133,20 @@ def process_commit_hashes(commit_hashes, repo_path, output_dir):
for hash in commit_hashes:
hash = hash.strip()
try:
subprocess.run(["git", "reset", "--hard"],
cwd=repo_path, check=True)
subprocess.run(["git", "reset", "--hard"], cwd=repo_path, check=True)
subprocess.run(["git", "clean", "-fdx"], cwd=repo_path, check=True)
# Checkout the specified hash
subprocess.run(["git", "checkout", hash],
cwd=repo_path, check=True)
subprocess.run(["git", "checkout", hash], cwd=repo_path, check=True)
print(f"Checked out {hash}")
# # Run Maven to build the project for each commit
# subprocess.run(
# ["mvn", "clean", "install", "-DskipTests", "--projects", ":hadoop-common", "--also-make"],
# cwd=repo_path,
# check=True
# )
# Get repo requirements
java_required, maven_required = parse_pom_xml(
os.path.join(repo_path, "pom.xml")
......@@ -188,10 +192,14 @@ def process_commit_hashes(commit_hashes, repo_path, output_dir):
)
print("=" * 10)
is_version_compatible = version_is_compatible(
java_required, java_installed)
is_version_compatible = version_is_compatible(java_required, java_installed)
java_versions[hash]["compatible"] = is_version_compatible
is_version_compatible = True # Temporarily skipping checking compatible versions
is_version_compatible = (
True # Temporarily skipping checking compatible versions
)
# Generate sources
subprocess.run(f"cd {repo_path} && mvn generate-sources", shell=True, check=True)
# Proceed if versions are compatible
if (
......@@ -200,12 +208,14 @@ def process_commit_hashes(commit_hashes, repo_path, output_dir):
# Run Maven to build the project
settings_path = create_maven_settings(repo_path)
subprocess.run(
f"cd {repo_path} && mvn --settings {settings_path} clean package -Dhttps.protocols=TLSv1.2 --projects :hadoop-common --also-make -DskipTests \
-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true",
shell=True,
check=True,
maven_command = (
f"cd {repo_path} && mvn --settings {settings_path} clean package "
f"-Dhttps.protocols=TLSv1.2 --projects :hadoop-common --also-make "
f"-DskipTests -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true "
f"-Dcommon.protobuf2.scope=compile"
)
subprocess.run(maven_command, shell=True, check=True)
# Copy the jar files to the specified output directory
copy_jar_files(
f"{repo_path}/hadoop-common-project/hadoop-common/target",
......@@ -225,6 +235,6 @@ def process_commit_hashes(commit_hashes, repo_path, output_dir):
def create_packages(repo_path, commit_info_file_path, output_dir):
# Open the file and load the data
with open(commit_info_file_path, "r") as file:
commit_hashes = [commit['commit_hash'] for commit in json.load(file)]
commit_hashes = [commit["commit_hash"] for commit in json.load(file)]
process_commit_hashes(commit_hashes, repo_path, output_dir)
src/hadoop_analysis/issue_commit_linkage.py
+ 18
− 0
View file @ 90ed6b56
......@@ -2,6 +2,7 @@ import os
import json
import logging
import requests
import subprocess
from pydriller import Repository
from commit_analyzer import commit_details
......@@ -43,6 +44,22 @@ def link_issues_to_commits(repo_path, issues_file, output_dir):
output = {issue["issueId"]: [] for issue in issues}
all_commits = []
try:
subprocess.run(["git", "reset", "--hard"],
cwd=repo_path, check=True)
subprocess.run(["git", "clean", "-fdx"], cwd=repo_path, check=True)
# Checkout the specified hash
subprocess.run(["git", "checkout", "trunk"],
cwd=repo_path, check=True)
print(f"Checked out {hash}")
except subprocess.CalledProcessError as e:
print(
f"Error: {e}. Command '{e.cmd}' returned non-zero exit status {e.returncode}."
)
for commit in Repository(repo_path).traverse_commits():
decision_properties = ['isExistenceDecision', 'isExecutiveDecision', 'isPropertyDecision']
matching_issues = get_matching_issues(commit, issues)
......@@ -61,6 +78,7 @@ def link_issues_to_commits(repo_path, issues_file, output_dir):
# Write to JSON files
print("Writing to", os.path.join(output_dir, "issue_commit_relationships.json"))
with open(os.path.join(output_dir, "issue_commit_relationships.json"), "w") as file:
json.dump(output, file, indent=4)
with open(os.path.join(output_dir, "detailed_commits_info.json"), "w") as file:
......
src/hadoop_analysis/visualization.py
+ 17
− 15
View file @ 90ed6b56
......@@ -4,18 +4,26 @@ import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.backends.backend_pdf import PdfPages
def load_data(json_path):
"""Load and transform data from a JSON file into a pandas DataFrame."""
"""Load data from a JSON file into a pandas DataFrame and derive decision types."""
with open(json_path, "r") as file:
data = json.load(file)
df = pd.DataFrame(data)
# Transform the dictionary to a list of values (commit details)
transformed_data = list(data.values())
# Define a function to determine decision type based on Boolean flags
def get_decision_type(row):
if row['isExistenceDecision']:
return 'Existence'
elif row['isExecutiveDecision']:
return 'Executive'
elif row['isPropertyDecision']:
return 'Property'
return 'Unknown' # Fallback if none of the flags are True
# Normalize the list of dictionaries to create a DataFrame
return pd.json_normalize(transformed_data)
# Apply the function to each row to create the 'Decision Type' column
df['Decision Type'] = df.apply(get_decision_type, axis=1)
return df
def create_boxplot(data, x, y, title, pdf):
"""Create a boxplot from the provided data and save to the PDF."""
......@@ -27,35 +35,29 @@ def create_boxplot(data, x, y, title, pdf):
plt.tight_layout()
plt.savefig(pdf, format="pdf")
def create_charts(json_file, output_dir):
data = load_data(json_file)
print(data.columns)
# Ensure DMM metrics are accessible by flattening the nested structure
data["DMM_unit_size"] = data["DMM_metrics.size"]
data["DMM_unit_complexity"] = data["DMM_metrics.complexity"]
data["DMM_unit_interfacing"] = data["DMM_metrics.interfacing"]
with PdfPages(f"{output_dir}/visualization_report.pdf") as pdf:
create_boxplot(
data,
"Decision Type",
"DMM_unit_size",
"dmm_size",
"Size Information per Decision Type",
pdf,
)
create_boxplot(
data,
"Decision Type",
"DMM_unit_complexity",
"dmm_complexity",
"Complexity Information per Decision Type",
pdf,
)
create_boxplot(
data,
"Decision Type",
"DMM_unit_interfacing",
"dmm_interfacing",
"Interfacing Information per Decision Type",
pdf,
)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment