Fix failure in unsafe-checker on invalid UTF-8

Author: davidlattimore

src/unsafe_checker.rs

@@ -8,9 +8,14 @@ use anyhow::Result;
 use std::path::Path;
 
 pub(crate) fn scan_path(path: &Path) -> Result<Option<UnsafeUsage>> {
-    let source = std::fs::read_to_string(path)
-        .with_context(|| format!("Failed to read `{}`", path.display()))?;
-    Ok(scan_string(&source, path))
+    let bytes =
+        std::fs::read(path).with_context(|| format!("Failed to read `{}`", path.display()))?;
+    let Ok(source) = std::str::from_utf8(&bytes) else {
+        // If the file isn't valid UTF-8 then we don't need to check it for the unsafe keyword,
+        // since it can't be a source file that the rust compiler would accept.
+        return Ok(None);
+    };
+    Ok(scan_string(source, path))
 }
 
 fn scan_string(source: &str, path: &Path) -> Option<UnsafeUsage> {

test_crates/crab2/src/lib.rs

@@ -1,7 +1,15 @@
+// Make sure that we don't add any unsafe to this crate. We want this crate to have no unsafe since
+// it has an include_bytes! below and we want to make sure that include_bytes! of invalid UTF-8
+// doesn't cause the unsafe token-based checker any problems. If we had some actual unsafe in this
+// crate, then we'd need to allow it and then the unsafe-checker wouldn't run.
+#![forbid(unsafe_code)]
+
 use bar::write as woozle;
 use fob::fs as bar;
 use std as fob;
 
+const DATA: &[u8] = include_bytes!("../../data/random.data");
+
 pub mod stuff {
     #[link(
         name = "nothing",
@@ -16,5 +24,8 @@ pub mod stuff {
             true
         });
         crab3::do_stuff();
+
+        let total: u32 = crate::DATA.iter().cloned().map(|byte| byte as u32).sum();
+        println!("{}", total);
     }
 }

test_crates/data/random.data

@@ -0,0 +1,2 @@
+��	%��r��
�^nV��5�\b���	�B�I�O@�M���S>��O�>Jr��@�x����!�T8\_X�u	���ߡ*��o�y�'�f��S��h��ca�
+*hx�-V}�$_��S�lY��rlKž,��ZA�F�� �?plmn��-�\b����` �&���ܞʝ<�
�zl�)6��Ha�`��q6�-�&P���2��I�)"� s΅��+���7��Q�˦�����bE�i��Ӟ'�Q�