diff --git a/src/context.rs b/src/context.rs
index 3bcff4d..267b7f4 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -102,7 +102,10 @@ where
     E: Debug,
 {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}\nCaused by: {:?}", self.context, self.error)
+        f.debug_struct("Error")
+            .field("context", &self.context.to_string())
+            .field("source", &self.error)
+            .finish()
     }
 }
 
diff --git a/src/error.rs b/src/error.rs
index 26990bc..8260e2a 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -678,6 +678,10 @@ impl ErrorImpl<()> {
     }
 
     fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if f.alternate() {
+            return Debug::fmt(self.error(), f);
+        }
+
         writeln!(f, "{}", self.error())?;
 
         let mut chain = self.chain().skip(1).enumerate().peekable();