diff --git a/pstoremem/metadata.go b/pstoremem/metadata.go
index 0548b47..a8c7d3c 100644
--- a/pstoremem/metadata.go
+++ b/pstoremem/metadata.go
@@ -7,15 +7,10 @@ import (
 	pstore "github.com/libp2p/go-libp2p-core/peerstore"
 )
 
-type metakey struct {
-	id  peer.ID
-	key string
-}
-
 type memoryPeerMetadata struct {
 	// store other data, like versions
 	// ds ds.ThreadSafeDatastore
-	ds       map[metakey]interface{}
+	ds       map[peer.ID]map[string]interface{}
 	dslock   sync.RWMutex
 	interned map[string]interface{}
 }
@@ -24,7 +19,7 @@ var _ pstore.PeerMetadata = (*memoryPeerMetadata)(nil)
 
 func NewPeerMetadata() *memoryPeerMetadata {
 	return &memoryPeerMetadata{
-		ds:       make(map[metakey]interface{}),
+		ds:       make(map[peer.ID]map[string]interface{}),
 		interned: make(map[string]interface{}),
 	}
 }
@@ -42,7 +37,12 @@ func (ps *memoryPeerMetadata) Put(p peer.ID, key string, val interface{}) error
 			ps.interned[vals] = val
 		}
 	}
-	ps.ds[metakey{p, key}] = val
+	m, ok := ps.ds[p]
+	if !ok {
+		m = make(map[string]interface{})
+		ps.ds[p] = m
+	}
+	m[key] = val
 	return nil
 }
 
@@ -52,9 +52,13 @@ func (ps *memoryPeerMetadata) Get(p peer.ID, key string) (interface{}, error) {
 	}
 	ps.dslock.RLock()
 	defer ps.dslock.RUnlock()
-	i, ok := ps.ds[metakey{p, key}]
+	m, ok := ps.ds[p]
 	if !ok {
 		return nil, pstore.ErrNotFound
 	}
-	return i, nil
+	val, ok := m[key]
+	if !ok {
+		return nil, pstore.ErrNotFound
+	}
+	return val, nil
 }