Check if Rust lib needs rebuilding. (#13759)

This protects against the common mistake of failing to remember to rebuild Rust code after making changes.
mv/cago-test-skippable
Erik Johnston 2022-09-12 11:03:42 +01:00 committed by GitHub
parent 4c4889cac0
commit ebfeac7c5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 149 additions and 1 deletions

1
changelog.d/13759.misc Normal file
View File

@ -0,0 +1 @@
Add a check for editable installs if the Rust library needs rebuilding.

View File

@ -19,3 +19,7 @@ name = "synapse.synapse_rust"
[dependencies] [dependencies]
pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] }
[build-dependencies]
blake2 = "0.10.4"
hex = "0.4.3"

45
rust/build.rs Normal file
View File

@ -0,0 +1,45 @@
//! This build script calculates the hash of all files in the `src/`
//! directory and adds it as an environment variable during build time.
//!
//! This is used so that the python code can detect when the built native module
//! does not match the source in-tree, helping to detect the case where the
//! source has been updated but the library hasn't been rebuilt.
use std::path::PathBuf;
use blake2::{Blake2b512, Digest};
fn main() -> Result<(), std::io::Error> {
let mut dirs = vec![PathBuf::from("src")];
let mut paths = Vec::new();
while let Some(path) = dirs.pop() {
let mut entries = std::fs::read_dir(path)?
.map(|res| res.map(|e| e.path()))
.collect::<Result<Vec<_>, std::io::Error>>()?;
entries.sort();
for entry in entries {
if entry.is_dir() {
dirs.push(entry)
} else {
paths.push(entry.to_str().expect("valid rust paths").to_string());
}
}
}
paths.sort();
let mut hasher = Blake2b512::new();
for path in paths {
let bytes = std::fs::read(path)?;
hasher.update(bytes);
}
let hex_digest = hex::encode(hasher.finalize());
println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}");
Ok(())
}

View File

@ -1,5 +1,13 @@
use pyo3::prelude::*; use pyo3::prelude::*;
/// Returns the hash of all the rust source files at the time it was compiled.
///
/// Used by python to detect if the rust library is outdated.
#[pyfunction]
fn get_rust_file_digest() -> &'static str {
env!("SYNAPSE_RUST_DIGEST")
}
/// Formats the sum of two numbers as string. /// Formats the sum of two numbers as string.
#[pyfunction] #[pyfunction]
#[pyo3(text_signature = "(a, b, /)")] #[pyo3(text_signature = "(a, b, /)")]
@ -11,6 +19,6 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
#[pymodule] #[pymodule]
fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?;
Ok(()) Ok(())
} }

View File

@ -1 +1,2 @@
def sum_as_string(a: int, b: int) -> str: ... def sum_as_string(a: int, b: int) -> str: ...
def get_rust_file_digest() -> str: ...

View File

@ -20,6 +20,8 @@ import json
import os import os
import sys import sys
from synapse.util.rust import check_rust_lib_up_to_date
# Check that we're not running on an unsupported Python version. # Check that we're not running on an unsupported Python version.
if sys.version_info < (3, 7): if sys.version_info < (3, 7):
print("Synapse requires Python 3.7 or above.") print("Synapse requires Python 3.7 or above.")
@ -78,3 +80,6 @@ if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
from synapse.util.patch_inline_callbacks import do_patch from synapse.util.patch_inline_callbacks import do_patch
do_patch() do_patch()
check_rust_lib_up_to_date()

84
synapse/util/rust.py Normal file
View File

@ -0,0 +1,84 @@
# Copyright 2022 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from hashlib import blake2b
import synapse
from synapse.synapse_rust import get_rust_file_digest
def check_rust_lib_up_to_date() -> None:
"""For editable installs check if the rust library is outdated and needs to
be rebuilt.
"""
if not _dist_is_editable():
return
synapse_dir = os.path.dirname(synapse.__file__)
synapse_root = os.path.abspath(os.path.join(synapse_dir, ".."))
# Double check we've not gone into site-packages...
if os.path.basename(synapse_root) == "site-packages":
return
# ... and it looks like the root of a python project.
if not os.path.exists("pyproject.toml"):
return
# Get the hash of all Rust source files
hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src"))
if hash != get_rust_file_digest():
raise Exception("Rust module outdated. Please rebuild using `poetry install`")
def _hash_rust_files_in_directory(directory: str) -> str:
"""Get the hash of all files in a directory (recursively)"""
directory = os.path.abspath(directory)
paths = []
dirs = [directory]
while dirs:
dir = dirs.pop()
with os.scandir(dir) as d:
for entry in d:
if entry.is_dir():
dirs.append(entry.path)
else:
paths.append(entry.path)
# We sort to make sure that we get a consistent and well-defined ordering.
paths.sort()
hasher = blake2b()
for path in paths:
with open(os.path.join(directory, path), "rb") as f:
hasher.update(f.read())
return hasher.hexdigest()
def _dist_is_editable() -> bool:
"""Is distribution an editable install?"""
for path_item in sys.path:
egg_link = os.path.join(path_item, "matrix-synapse.egg-link")
if os.path.isfile(egg_link):
return True
return False