fix: custom instOfHandler result should be cast to boolean (#10197)

packages/babel-plugin-transform-instanceof/test/fixtures/instanceof/instanceof/exec.js

@@ -4,6 +4,11 @@ foo[Symbol.hasInstance]= function () { return true; };
 var bar = {};
 
 expect(bar instanceof foo).toBe(true);
+
+var qux = {};
+qux[Symbol.hasInstance]= function () { return NaN };
+expect(bar instanceof qux).toBe(false);
+
 expect(new String).toBeInstanceOf(String);
 
 //