diff --git a/src/hb-object-private.hh b/src/hb-object-private.hh
index 75a72f4..1f61dce 100644
--- a/src/hb-object-private.hh
+++ b/src/hb-object-private.hh
@@ -62,7 +62,6 @@
 
 /* user_data */
 
-#define HB_USER_DATA_ARRAY_INIT {HB_MUTEX_INIT, HB_LOCKABLE_SET_INIT}
 struct hb_user_data_array_t
 {
   struct hb_user_data_item_t {
@@ -97,9 +96,9 @@
 struct hb_object_header_t
 {
   hb_reference_count_t ref_count;
-  hb_user_data_array_t user_data;
+  hb_user_data_array_t *user_data;
 
-#define HB_OBJECT_HEADER_STATIC {HB_REFERENCE_COUNT_INIT, HB_USER_DATA_ARRAY_INIT}
+#define HB_OBJECT_HEADER_STATIC {HB_REFERENCE_COUNT_INIT, nullptr}
 
   private:
   ASSERT_POD ();
@@ -133,7 +132,7 @@
 static inline void hb_object_init (Type *obj)
 {
   obj->header.ref_count.init (1);
-  obj->header.user_data.init ();
+  obj->header.user_data = nullptr;
 }
 template <typename Type>
 static inline bool hb_object_is_inert (const Type *obj)
@@ -172,7 +171,11 @@
 static inline void hb_object_fini (Type *obj)
 {
   obj->header.ref_count.fini (); /* Do this before user_data */
-  obj->header.user_data.fini ();
+  if (obj->header.user_data)
+  {
+    obj->header.user_data->fini ();
+    free (obj->header.user_data);
+  }
 }
 template <typename Type>
 static inline bool hb_object_set_user_data (Type               *obj,
@@ -184,17 +187,34 @@
   if (unlikely (!obj || hb_object_is_inert (obj)))
     return false;
   assert (hb_object_is_valid (obj));
-  return obj->header.user_data.set (key, data, destroy, replace);
+
+retry:
+  hb_user_data_array_t *user_data = (hb_user_data_array_t *) hb_atomic_ptr_get (&obj->header.user_data);
+  if (unlikely (!user_data))
+  {
+    user_data = (hb_user_data_array_t *) calloc (sizeof (hb_user_data_array_t), 1);
+    if (unlikely (!user_data))
+      return false;
+    user_data->init ();
+    if (unlikely (!hb_atomic_ptr_cmpexch (&obj->header.user_data, nullptr, user_data)))
+    {
+      user_data->fini ();
+      free (user_data);
+      goto retry;
+    }
+  }
+
+  return user_data->set (key, data, destroy, replace);
 }
 
 template <typename Type>
 static inline void *hb_object_get_user_data (Type               *obj,
 					     hb_user_data_key_t *key)
 {
-  if (unlikely (!obj || hb_object_is_inert (obj)))
+  if (unlikely (!obj || hb_object_is_inert (obj) || !obj->header.user_data))
     return nullptr;
   assert (hb_object_is_valid (obj));
-  return obj->header.user_data.get (key);
+  return obj->header.user_data->get (key);
 }